framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,1,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,3,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,3,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,15,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,15,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,127,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,255,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,511,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,511,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,1023,0.12595199545224509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,1023,0.1256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,1023,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,1023,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,1023,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,2047,0.24302933613459268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,2047,0.24712532758712769
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,2047,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,2047,0.24183466037114462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,2047,0.24166399240493774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,2047,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,2047,0.24285866816838583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,2047,0.24183466037114462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,2047,0.24166399240493774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,4095,0.47650134563446045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,4095,0.4766720136006673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,4095,0.4775253136952718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,4095,0.4771840174992879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,4095,0.475818673769633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,4095,0.47598934173583984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,4095,0.4766720136006673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,4095,0.475818673769633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,4095,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,4095,0.4768426815668742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,4095,0.4775253136952718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,4095,0.4771840174992879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,4095,0.47598934173583984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,4095,0.475818673769633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,8191,0.9446400006612142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,8191,0.9446400006612142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,8191,0.9471999804178873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,8191,0.9461759726206461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,8191,0.9446400006612142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,8191,0.9441280364990234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,8191,0.9446400006612142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,8191,0.9446400006612142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,8191,0.9444693724314371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,8191,0.9460053443908691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,8191,0.9456640084584554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,16383,1.8809173901875813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,16383,1.8809173901875813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,16383,1.8812586466471355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,16383,1.8810879389444988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,16383,1.8812586466471355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,16383,1.8804052670796711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,16383,1.8851839701334636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,16383,1.881600062052409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,16383,1.8863786061604817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,16383,1.8809173901875813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,16383,1.8814293543497722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,16383,1.8812586466471355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,16383,1.8805759747823079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,16383,1.8812586466471355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,32767,3.779754638671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,32767,3.782997449239095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,32767,3.884544054667155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,32767,3.7806078592936196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,32767,3.884544054667155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,32767,3.8806187311808267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,32767,3.879253387451172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,32767,3.8920532862345376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,32767,3.7795839309692383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,32767,3.78163210550944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,32767,3.884373346964518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,32767,3.780266761779785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,32767,3.884544054667155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,32767,3.8809601465861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,65535,7.773013432820638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,65535,7.927808125813802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,65535,7.758848190307617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,65535,7.91978645324707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,65535,7.933610916137695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,65535,7.759018580118815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,65535,7.776767730712891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,65535,7.760213216145833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,65535,7.760042826334636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,65535,7.9291731516520185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,65535,7.922005335489909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,65535,7.9360001881917315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,65535,7.757312138875325
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,65535,7.750485102335612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,1,131071,15.546026865641275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,2,131071,15.53237279256185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,4,131071,15.498069763183594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,8,131071,15.841280619303385
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,16,131071,15.829503377278646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,32,131071,15.867562611897787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1,1,64,131071,15.508650461832682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,2,131071,15.517013549804688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,4,131071,15.518719991048178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,1,131071,15.539541880289713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,8,131071,15.49721654256185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,16,131071,15.844010670979818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,32,131071,15.832575480143229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1,1,64,131071,15.866709391276041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,3,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,15,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,63,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,127,0.022015998760859173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,255,0.03669333209594091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,511,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,511,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,1023,0.12578133742014566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,1023,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,1023,0.1256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,1023,0.12578133742014566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,1023,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,1023,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,2047,0.24285866816838583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,2047,0.24251733223597208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,2047,0.24183466037114462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,2047,0.24251733223597208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,2047,0.24285866816838583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,4095,0.4766720136006673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,4095,0.4773546854654948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,4095,0.47650134563446045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,4095,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,4095,0.4771840174992879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,4095,0.4763306776682536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,4095,0.4773546854654948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,4095,0.4766720136006673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,4095,0.4773546854654948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,4095,0.47598934173583984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,4095,0.47598934173583984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,4095,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,4095,0.4776959816614787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,4095,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,8191,0.947370688120524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,8191,0.947370688120524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,8191,0.9441280364990234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,8191,0.9466880162556967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,8191,0.9449813365936279
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,8191,0.9468586444854736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,8191,0.9466880162556967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,8191,0.9446400006612142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,8191,0.9471999804178873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,8191,0.9439573287963867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,8191,0.9442986647288004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,16383,2.0078934033711753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,16383,1.993557294209798
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,16383,1.9512319564819336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,16383,1.9891200065612793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,16383,1.9908266067504883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,16383,1.988437334696452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,16383,1.9964586893717449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,16383,1.9481600125630696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,16383,1.9505492846171062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,16383,2.0039679209391275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,16383,1.9911680221557617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,16383,1.9898026784261067
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,16383,1.9903146425882976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,16383,1.9875839551289876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,32767,4.00930118560791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,32767,3.8947839736938477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,32767,3.8935893376668296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,32767,3.971242586771647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,32767,3.970730781555176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,32767,3.9729493459065757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,32767,3.98796812693278
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,32767,3.9862613677978516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,32767,3.9915520350138345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,32767,3.8920532862345376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,32767,3.890517234802246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,32767,3.971242586771647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,32767,3.9729493459065757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,32767,3.969877243041992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,65535,7.969109217325847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,65535,7.962282816569011
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,65535,7.932757059733073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,65535,7.77181879679362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,65535,7.778303782145183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,65535,7.940949122111003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,65535,7.929685592651367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,65535,7.928490956624349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,65535,7.961599985758464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,65535,7.97764269510905
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,65535,7.921663920084636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,65535,7.7707945505778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,65535,7.940266927083333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,65535,7.775744120279948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,1,131071,15.94589869181315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,4,131071,15.912789662679037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,2,131071,15.854420979817709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,8,131071,15.85971196492513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,16,131071,15.848276774088541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,2,1,64,131071,15.879508972167969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,32,131071,15.536127726236979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,1,131071,15.597396850585938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,2,131071,15.863807678222656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,4,131071,15.912960052490234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,8,131071,15.835989634195963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,16,131071,15.860394795735678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,2,1,64,131071,15.878826141357422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,32,131071,15.877119700113932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,7,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,15,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,63,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,127,0.022015998760859173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,255,0.03669333209594091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,255,0.03669333209594091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,511,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,1023,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,1023,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,1023,0.1360213359196981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,1023,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,1023,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,2047,0.24285866816838583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,2047,0.24285866816838583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,2047,0.24251733223597208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,2047,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,2047,0.24251733223597208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,2047,0.24183466037114462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,2047,0.24251733223597208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,2047,0.24183466037114462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,4095,0.4775253136952718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,4095,0.4766720136006673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,4095,0.4766720136006673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,4095,0.4775253136952718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,4095,0.4763306776682536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,4095,0.4773546854654948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,4095,0.4771840174992879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,4095,0.47650134563446045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,4095,0.4768426815668742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,4095,0.4775253136952718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,4095,0.47650134563446045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,4095,0.475818673769633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,4095,0.47598934173583984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,4095,0.47598934173583984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,8191,1.0134186744689941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,8191,1.006592035293579
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,8191,0.9997653166453043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,8191,1.006933371225993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,8191,1.006592035293579
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,8191,1.0074453353881836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,8191,1.0335573355356853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,8191,1.006933371225993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,8191,1.005226691563924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,8191,0.9997653166453043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,8191,1.0006186962127686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,8191,1.0064213275909424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,8191,1.0062506993611653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,8191,1.001471996307373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,16383,2.0089173316955566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,16383,2.0066986083984375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,16383,2.0118187268575034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,16383,1.9908266067504883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,16383,2.0089173316955566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,16383,2.008234659830729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,16383,1.9928746223449707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,16383,2.0077226956685386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,16383,2.0078934033711753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,16383,2.0096000035603843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,16383,1.997312068939209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,16383,1.9927040735880535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,16383,2.0085760752360025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,16383,2.0107946395874023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,32767,3.9910399119059243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,32767,3.9789225260416665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,32767,3.998719851175944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,32767,4.002986590067546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,32767,3.9697065353393555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,32767,3.9869438807169595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,32767,4.004522641499837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,32767,4.027733484903972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,32767,3.974656105041504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,32767,3.976703961690267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,32767,3.9982080459594727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,32767,3.9702186584472656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,32767,4.0038401285807295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,32767,3.9862613677978516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,65535,8.056661605834961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,65535,8.011605580647787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,65535,7.93343989054362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,65535,7.993343989054362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,65535,7.928831736246745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,65535,7.93992551167806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,65535,8.02781867980957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,65535,8.016554514567057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,65535,8.007509231567383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,65535,8.003583908081055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,65535,7.928661346435547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,65535,7.93446413675944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,65535,7.9406077067057295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,65535,7.993685404459636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,2,131071,16.01416524251302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,8,131071,15.985664367675781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,4,131071,16.005631764729817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,1,131071,16.009557088216145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,32,131071,15.864149729410807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,16,131071,15.859029134114584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,1,131071,16.053760528564453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,4,1,64,131071,15.878997802734375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,2,131071,16.01467768351237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,4,131071,15.992661794026693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,8,131071,15.997269948323568
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,16,131071,15.90869394938151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,4,1,64,131071,15.854420979817709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,32,131071,15.864490509033203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,1,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,3,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,7,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,7,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,15,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,63,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,127,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,127,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,127,0.022015998760859173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,127,0.022015998760859173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,255,0.03669333209594091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,255,0.03669333209594091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,511,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,511,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,1023,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,1023,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,1023,0.12595199545224509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,1023,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,2047,0.24302933613459268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,2047,0.24302933613459268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,1023,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,2047,0.24183466037114462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,2047,0.24183466037114462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,2047,0.24183466037114462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,2047,0.24302933613459268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,2047,0.24337067206700644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,2047,0.24183466037114462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,2047,0.24149332443873087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,2047,0.24166399240493774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,4095,0.5109759966532389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,4095,0.5097813208897909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,4095,0.5082453489303589
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,4095,0.5073920090993246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,4095,0.5082453489303589
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,4095,0.5073920090993246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,4095,0.5126826763153076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,4095,0.5077333450317383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,4095,0.5090986490249634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,4095,0.5089279810587565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,4095,0.5087573528289795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,4095,0.5087573528289795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,4095,0.5082453489303589
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,8191,1.0190506776173909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,4095,0.5084160168965658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,8191,1.0091520150502522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,8191,1.0144426822662354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,8191,1.0120533307393391
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,8191,1.0115413665771484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,8191,1.01256529490153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,8191,1.0141013463338215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,8191,1.0437973340352376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,8191,1.0122240384419758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,8191,1.0084693431854248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,8191,1.0113706588745117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,8191,1.012394666671753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,8191,1.0100053151448567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,8191,1.0122240384419758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,16383,2.0293973286946616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,16383,2.020693302154541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,16383,2.0176213582356772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,16383,2.0159145991007485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,16383,2.012330691019694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,16383,2.0164267222086587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,16383,2.013696034749349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,16383,2.026495933532715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,16383,2.018986701965332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,16383,2.018986701965332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,16383,2.0152319272359214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,16383,2.0135253270467124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,16383,2.0130133628845215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,32767,4.040021260579427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,16383,2.0176213582356772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,32767,4.0321706136067705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,32767,4.0272213617960615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,32767,4.029952049255371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,32767,4.021589279174805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,32767,4.012373288472493
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,32767,4.018687884012858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,32767,4.041215896606445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,32767,4.029952049255371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,32767,4.033194541931152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,32767,4.023978551228841
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,32767,4.029439926147461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,32767,4.021247863769531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,32767,4.010154724121094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,65535,8.03874142964681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,65535,8.042837142944336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,65535,8.046080271402994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,65535,8.067754745483398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,65535,8.036693572998047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,65535,8.0447146097819
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,65535,8.034645080566406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,65535,8.060415903727213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,65535,8.037546793619791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,65535,8.0447146097819
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,65535,8.037888209025065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,65535,8.048128128051758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,65535,8.03054936726888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,65535,8.042154947916666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,1,131071,16.119125366210938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,2,131071,16.03276824951172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,4,131071,16.045055389404297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,8,131071,16.08448028564453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,16,131071,16.058367411295574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,32,131071,16.07799530029297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,8,1,64,131071,16.04317855834961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,2,131071,16.062122344970703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,1,131071,16.133973439534504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,4,131071,16.026453653971355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,8,131071,16.04625066121419
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,16,131071,16.085163116455078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,32,131071,16.057003021240234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,8,1,64,131071,16.077653249104817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,3,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,15,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,15,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,63,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,511,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,511,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,1023,0.12612266341845194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,1023,0.12578133742014566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,1023,0.12578133742014566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,1023,0.12578133742014566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,1023,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,1023,0.1262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,1023,0.12612266341845194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,1023,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,1023,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,2047,0.2611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,2047,0.25941334168116253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,2047,0.2599253257115682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,2047,0.25890133778254193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,2047,0.2583893338839213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,2047,0.258730669816335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,2047,0.25890133778254193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,2047,0.26146133740743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,2047,0.25941334168116253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,2047,0.25941334168116253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,2047,0.25924267371495563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,2047,0.258730669816335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,2047,0.2583893338839213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,2047,0.2583893338839213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,4095,0.5104639927546183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,4095,0.5147306521733602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,4095,0.5101226568222046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,4095,0.5116586685180664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,4095,0.5102933247884115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,4095,0.5108053286870321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,4095,0.5143893162409464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,4095,0.5097813208897909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,4095,0.5109759966532389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,4095,0.5113173325856527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,4095,0.5087573528289795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,4095,0.5109759966532389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,4095,0.5102933247884115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,8191,1.020416021347046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,4095,0.5101226568222046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,8191,1.0154666900634766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,8191,1.0190506776173909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,8191,1.009493350982666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,8191,1.016319990158081
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,8191,1.0134186744689941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,8191,1.0212693214416504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,8191,1.0154666900634766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,8191,1.016319990158081
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,8191,1.0147840181986492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,8191,1.0159786542256672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,8191,1.0098346869150798
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,8191,1.0164906978607178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,16383,2.0307626724243164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,8191,1.0142719745635986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,16383,2.0213759740193686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,16383,2.024789333343506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,16383,2.020352045694987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,16383,2.0215466817220054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,16383,2.0169386863708496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,16383,2.0224000612894693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,16383,2.030933380126953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,16383,2.0205225944519043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,16383,2.0215466817220054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,16383,2.0232532819112143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,16383,2.0215466817220054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,16383,2.018474737803141
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,16383,2.015061378479004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,32767,4.058794657389323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,32767,4.030463854471843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,32767,4.029439926147461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,32767,4.031829198201497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,32767,4.0352427164713545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,32767,4.034389177958171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,32767,4.029439926147461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,32767,4.041386604309082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,32767,4.028245290120442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,32767,4.027733484903972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,32767,4.032512029012044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,32767,4.042069435119629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,32767,4.035072008768718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,32767,4.035072008768718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,65535,8.092159907023111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,65535,8.048298517862955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,65535,8.065877278645834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,65535,8.047274907430014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,65535,8.055466969807943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,65535,8.063488006591797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,65535,8.036693572998047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,65535,8.097450892130533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,65535,8.054954528808594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,65535,8.047615687052408
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,65535,8.055807749430338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,65535,8.04573885599772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,65535,8.053759892781576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,65535,8.063658396402994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,1,131071,16.15342966715495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,2,131071,16.111104329427082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,4,131071,16.08857599894206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,8,131071,16.079530080159504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,16,131071,16.110591888427734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,32,131071,16.076800028483074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,16,1,64,131071,16.09386698404948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,1,131071,16.186197916666668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,2,131071,16.07099787394206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,1,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,4,131071,16.10052235921224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,8,131071,16.084821065266926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,16,131071,16.083114624023438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,32,131071,16.111615498860676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,16,1,64,131071,16.077653249104817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,1,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,3,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,3,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,3,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,7,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,7,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,15,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,15,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,31,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,31,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,63,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,63,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,255,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,255,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,511,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,511,0.06809600194295247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,511,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,511,0.07048533360163371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,511,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,511,0.06758399804433186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,1023,0.13738666971524557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,1023,0.13943466544151306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,1023,0.1365333298842112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,1023,0.1358506679534912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,1023,0.1346560021241506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,1023,0.13482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,1023,0.13550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,1023,0.1365333298842112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,1023,0.13875200351079306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,1023,0.13772799571355185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,1023,0.13567999998728433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,1023,0.1353386640548706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,1023,0.13448533415794373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,1023,0.13499733805656433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,2047,0.2667520046234131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,2047,0.2645333409309387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,2047,0.26282666126887005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,2047,0.2624853253364563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,2047,0.2744320034980774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,2047,0.2619733413060506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,2047,0.2606079975763957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,2047,0.26419200499852497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,2047,0.2676053245862325
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,2047,0.2624853253364563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,2047,0.26231465737024945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,2047,0.26265599330266315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,2047,0.26265599330266315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,2047,0.26180267333984375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,4095,0.5167786677678426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,4095,0.520362655321757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,4095,0.5160959959030151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,4095,0.5152426560719808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,4095,0.514901320139567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,4095,0.5147306521733602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,4095,0.514901320139567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,4095,0.5208746592203776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,4095,0.517632007598877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,4095,0.5154133240381876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,4095,0.5164373318354288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,4095,0.5143893162409464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,4095,0.5145599842071533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,4095,0.5147306521733602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,8191,1.0316800276438396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,8191,1.0238293011983235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,8191,1.0233173370361328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,8191,1.0222933292388916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,8191,1.0197333494822185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,8191,1.021781365076701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,8191,1.0209279855092366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,8191,1.0299733479817708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,8191,1.0224640369415283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,8191,1.0246826807657878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,8191,1.022976001103719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,8191,1.0222933292388916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,8191,1.0188799699147542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,8191,1.0207573572794597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,16383,2.0365653038024902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,16383,2.045952002207438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,16383,2.0363945960998535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,16383,2.0365653038024902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,16383,2.035370667775472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,16383,2.034005324045817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,16383,2.033151944478353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,16383,2.0473173459370932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,16383,2.0375893910725913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,16383,2.0334933598836265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,16383,2.035029411315918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,16383,2.037077267964681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,16383,2.0351999600728354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,16383,2.035029411315918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,32767,4.060160001118978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,32767,4.0763734181722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,32767,4.054357210795085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,32767,4.055381457010905
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,32767,4.058282534281413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,32767,4.05845324198405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,32767,4.057941436767578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,32767,4.084906578063965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,32767,4.060672124226888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,32767,4.052138646443685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,32767,4.05350399017334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,32767,4.05350399017334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,32767,4.058112144470215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,32767,4.05674680074056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,65535,8.156842549641928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,65535,8.112810770670572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,65535,8.100351969401041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,65535,8.084480285644531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,65535,8.102912267049154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,65535,8.097450892130533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,65535,8.105813344319662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,65535,8.154282887776693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,65535,8.14353052775065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,65535,8.103765487670898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,65535,8.097450892130533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,65535,8.084992090861002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,65535,8.10052235921224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,65535,8.094207763671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,1,131071,16.291156768798828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,2,131071,16.224085489908855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,4,131071,16.200363159179688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,8,131071,16.19387690226237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,16,131071,16.182100931803387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,32,131071,16.169301350911457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,32,1,64,131071,16.192000071207683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,1,131071,16.272383371988933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,1,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,2,131071,16.215211232503254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,4,131071,16.200191497802734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,1,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,8,131071,16.193365732828777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,1,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,16,131071,16.19950993855794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,32,131071,16.181930541992188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,32,1,64,131071,16.166912078857422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,1,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,1,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,1,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,1,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,1,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,3,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,1,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,3,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,3,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,3,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,3,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,3,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,3,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,3,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,7,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,7,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,7,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,7,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,7,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,7,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,7,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,7,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,7,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,7,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,7,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,7,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,15,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,15,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,15,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,15,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,15,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,15,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,15,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,15,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,15,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,15,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,15,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,15,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,15,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,31,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,31,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,31,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,31,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,31,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,31,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,31,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,31,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,31,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,31,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,31,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,31,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,31,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,31,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,63,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,63,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,63,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,63,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,63,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,63,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,63,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,63,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,63,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,63,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,63,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,63,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,63,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,127,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,127,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,127,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,127,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,127,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,63,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,127,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,127,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,127,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,127,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,127,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,127,0.0288426677385966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,127,0.0288426677385966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,127,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,127,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,255,0.05358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,255,0.05085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,255,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,255,0.05034666756788889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,255,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,255,0.04983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,255,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,255,0.05085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,255,0.053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,255,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,255,0.050517335534095764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,255,0.05000533163547516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,255,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,255,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,511,0.09745066364606221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,511,0.09642666578292847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,511,0.0962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,511,0.09608532985051473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,511,0.09540266791979472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,511,0.09523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,511,0.09506133198738098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,511,0.09745066364606221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,511,0.09659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,511,0.09608532985051473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,511,0.09540266791979472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,511,0.09557333588600159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,511,0.09540266791979472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,511,0.09523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,1023,0.18193066120147705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,1023,0.1807360053062439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,1023,0.1807360053062439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,1023,0.1812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,1023,0.18056533734003702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,1023,0.1812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,1023,0.1812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,1023,0.18158932526906332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,1023,0.1807360053062439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,1023,0.18107734123865762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,1023,0.18090667327245077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,1023,0.18107734123865762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,1023,0.18090667327245077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,1023,0.1812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,2047,0.3515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,2047,0.35072000821431476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,2047,0.35140268007914227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,2047,0.35072000821431476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,2047,0.35174401601155597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,2047,0.3519146839777629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,2047,0.35225598017374676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,2047,0.3515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,2047,0.3515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,2047,0.35174401601155597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,2047,0.3515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,2047,0.3515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,2047,0.35174401601155597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,2047,0.3524266481399536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,4095,0.6881279945373535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,4095,0.6927359898885092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,4095,0.6901760101318359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,4095,0.6930773258209229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,4095,0.6929066975911459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,4095,0.6932480335235596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,4095,0.6934186617533366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,4095,0.6886400381724039
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,4095,0.6896639664967855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,4095,0.6929066975911459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,4095,0.702293316523234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,4095,0.6930773258209229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,4095,0.6934186617533366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,4095,0.6937599976857504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,8191,1.3591893513997395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,8191,1.3707946141560872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,8191,1.366528034210205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,8191,1.3760852813720703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,8191,1.3786452611287434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,8191,1.3765974044799805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,8191,1.3760852813720703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,8191,1.3603839874267578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,8191,1.3716479937235515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,8191,1.375402609507243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,8191,1.3646507263183594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,8191,1.377962589263916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,8191,1.3800106048583984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,8191,1.377621332804362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,16383,2.7047252655029297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,16383,2.7281068166097007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,16383,2.715306599934896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,16383,2.737834612528483
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,16383,2.74397881825765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,16383,2.7496105829874673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,16383,2.738346735636393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,16383,2.702335993448893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,16383,2.717013359069824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,16383,2.7269118626912436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,16383,2.7347625096639
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,16383,2.7415892283121743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,16383,2.7465387980143228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,16383,2.749781290690104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,32767,5.419178644816081
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,32767,5.385898590087891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,32767,5.458943684895833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,32767,5.439999898274739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,32767,5.466794967651367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,32767,5.476352055867513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,32767,5.482666651407878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,32767,5.389312108357747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,32767,5.417130788167317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,32767,5.4586029052734375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,32767,5.4662825266520185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,32767,5.439829508463542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,32767,5.469354629516602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,32767,5.478570938110352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,65535,10.783573150634766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,65535,10.810197194417318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,65535,10.864468892415365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,65535,10.902357737223307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,65535,10.930346171061197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,65535,10.925739288330078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,65535,10.93051783243815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,65535,10.836650848388672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,65535,10.895701090494791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,65535,10.753194173177084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,65535,10.902015686035156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,65535,10.921642303466797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,65535,10.922495524088541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,65535,10.929835001627604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,1,131071,21.470550537109375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,2,131071,21.65674591064453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,4,131071,21.77484893798828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,16,131071,21.835433959960938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,32,131071,21.843114217122395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,64,1,64,131071,22.02538553873698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,1,131071,21.47857157389323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,8,131071,21.800277709960938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,1,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,1,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,1,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,2,131071,21.625343322753906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,1,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,1,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,4,131071,21.7533442179362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,16,131071,21.837310791015625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,1,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,8,131071,21.850794474283855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,32,131071,21.846356709798176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,64,1,64,131071,22.043136596679688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,1,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,1,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,1,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,1,0.031914666295051575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,1,0.02867199977238973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,3,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,3,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,3,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,3,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,3,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,3,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,3,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,3,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,3,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,3,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,3,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,3,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,7,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,7,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,7,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,7,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,7,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,7,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,7,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,7,0.03276800115903219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,7,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,7,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,7,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,7,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,15,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,15,0.031914666295051575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,15,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,15,0.031744000812371574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,15,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,7,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,15,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,15,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,15,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,15,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,15,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,15,0.032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,15,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,15,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,31,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,31,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,31,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,31,0.031914666295051575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,15,0.0288426677385966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,31,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,31,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,31,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,31,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,31,0.032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,31,0.031914666295051575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,31,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,31,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,31,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,31,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,63,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,63,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,63,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,63,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,63,0.0288426677385966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,63,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,63,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,63,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,63,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,63,0.03566933423280716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,63,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,63,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,63,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,63,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,127,0.056661332647005715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,127,0.04795733094215393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,127,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,127,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,127,0.04471466441949209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,127,0.04454400142033895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,127,0.04471466441949209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,127,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,127,0.0481279989083608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,127,0.04471466441949209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,127,0.04539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,127,0.04471466441949209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,127,0.04454400142033895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,127,0.056832000613212585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,255,0.0897706647713979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,255,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,255,0.08533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,255,0.08482133348782857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,255,0.0846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,255,0.08430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,255,0.09403733412424724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,255,0.0897706647713979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,255,0.0865280032157898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,255,0.08550399541854858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,255,0.08482133348782857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,255,0.08430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,255,0.09983999530474345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,255,0.08413867155710857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,511,0.1629866659641266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,511,0.16025599837303162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,511,0.15923200050989786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,511,0.1687893271446228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,511,0.15820800264676413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,511,0.15837867061297098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,511,0.16895999511082968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,511,0.16554666558901468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,511,0.15940266847610474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,511,0.16025599837303162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,511,0.15837867061297098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,511,0.15786666671435037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,511,0.15820800264676413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,1023,0.3155626654624939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,1023,0.30958932638168335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,1023,0.3068586587905884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,1023,0.30532266696294147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,1023,0.3031040032704671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,511,0.1585493286450704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,1023,0.30429865916570026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,1023,0.30395734310150146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,1023,0.30907734235127765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,1023,0.31573333342870075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,1023,0.3063466747601827
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,1023,0.3054933349291484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,1023,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,1023,0.30395734310150146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,2047,0.6072320143381754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,2047,0.5981866518656412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,2047,0.5966506799062093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,2047,0.5957973400751749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,2047,0.5947733322779337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,2047,0.5939199924468994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,1023,0.30498133103052777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,2047,0.5954560041427612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,2047,0.6075733502705892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,2047,0.5986986557642618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,2047,0.5966506799062093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,2047,0.5956266721089681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,2047,0.5951146682103475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,2047,0.5947733322779337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,4095,1.1890347003936768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,4095,1.1758933067321777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,4095,1.1803306738535564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,4095,1.1775999863942463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,4095,1.1781120300292969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,4095,1.1770880222320557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,4095,1.179136037826538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,2047,0.5954560041427612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,4095,1.1787947018941243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,4095,1.1898880004882812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,4095,1.1799893379211426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,4095,1.1772586504618328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,4095,1.1774293581644695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,4095,1.1779413223266602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,4095,1.1779413223266602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,8191,2.347520033518473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,8191,2.3637332916259766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,8191,2.3471786181131997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,8191,2.3512746493021646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,8191,2.3488853772481284
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,8191,2.345301310221354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,8191,2.344618638356527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,8191,2.3625386555989585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,8191,2.3483732541402182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,8191,2.3522987365722656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,8191,2.3483732541402182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,8191,2.3466666539510093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,8191,2.3470080693562827
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,16383,4.683093388875325
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,16383,4.7247358957926435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,8191,2.3512746493021646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,16383,4.689237276713054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,16383,4.690090815226237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,16383,4.690261205037435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,16383,4.687701225280762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,16383,4.683264096577962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,16383,4.72268803914388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,16383,4.691455841064453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,16383,4.68667729695638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,16383,4.69162654876709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,16383,4.689237276713054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,16383,4.691114743550618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,16383,4.687701225280762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,32767,9.429333368937174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,32767,9.36618677775065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,32767,9.358847935994467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,32767,9.377279917399088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,32767,9.376938501993815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,32767,9.36618677775065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,32767,9.387349446614584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,32767,9.424896240234375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,32767,9.372159957885742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,32767,9.36635716756185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,32767,9.368746439615885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,32767,9.375232060750326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,32767,9.373013178507486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,32767,9.374207814534506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,65535,18.703360239664715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,65535,18.71308771769206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,65535,18.68731689453125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,65535,18.74551518758138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,65535,18.722644805908203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,65535,18.911744435628254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,65535,18.74278386433919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,65535,18.86515172322591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,65535,18.7151362101237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,65535,18.736469268798828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,65535,18.740564982096355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,65535,18.726229349772137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,65535,18.706261952718098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,65535,18.74875768025716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,2,131071,37.40415954589844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,1,131071,37.77672576904297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,4,131071,37.40620676676432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,8,131071,37.49427286783854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,32,131071,37.570048014322914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,16,131071,37.4847157796224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,128,1,64,131071,37.498026529947914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,1,131071,37.697364807128906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,1,0.055125330885251365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,1,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,1,0.04727466901143392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,1,0.045226668318112694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,1,0.04437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,1,0.04386133452256521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,1,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,1,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,1,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,1,0.04693333307902018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,1,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,1,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,1,0.04334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,8,131071,37.430442810058594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,1,0.04386133452256521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,16,131071,37.471232096354164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,32,131071,37.60315704345703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,3,0.05563733478387197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,3,0.04727466901143392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,3,0.045909335215886436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,3,0.04539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,2,131071,37.40313720703125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,3,0.04420266548792521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,128,1,64,131071,37.497344970703125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,3,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,3,0.04386133452256521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,3,0.046762665112813316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,3,0.055125330885251365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,3,0.045567999283472695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,3,0.04539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,4,131071,37.45416514078776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,3,0.04420266548792521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,3,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,7,0.05580799778302511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,7,0.045567999283472695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,7,0.046762665112813316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,3,0.04437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,7,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,7,0.04437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,7,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,7,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,7,0.05580799778302511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,7,0.04693333307902018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,7,0.04420266548792521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,7,0.045226668318112694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,7,0.04420266548792521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,7,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,7,0.04420266548792521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,15,0.04744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,15,0.04539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,15,0.05614933371543884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,15,0.045226668318112694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,15,0.04386133452256521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,15,0.04420266548792521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,15,0.05649066468079885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,15,0.04761599997679392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,15,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,15,0.04386133452256521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,15,0.04488533238569895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,15,0.04386133452256521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,15,0.04539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,15,0.04386133452256521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,31,0.07389866809050243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,31,0.045226668318112694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,31,0.04488533238569895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,31,0.0580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,31,0.04386133452256521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,31,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,31,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,31,0.0481279989083608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,31,0.04488533238569895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,31,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,31,0.04386133452256521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,31,0.05819733440876007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,31,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,31,0.045226668318112694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,63,0.06229333579540253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,63,0.05239466826121012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,63,0.04607999821503957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,63,0.043178667624791466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,63,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,63,0.04334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,63,0.04642133414745331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,63,0.05171200136343638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,63,0.04437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,63,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,63,0.06229333579540253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,63,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,63,0.04334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,63,0.043178667624791466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,127,0.08140799899895985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,127,0.09079466263453166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,127,0.07509333391984303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,127,0.07679999868075053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,127,0.0721919983625412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,127,0.07253333429495494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,127,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,127,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,127,0.08123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,127,0.07526400188604991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,127,0.07202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,127,0.0718506673971812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,127,0.09215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,127,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,255,0.13431466619173685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,255,0.13772799571355185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,255,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,255,0.1302186648050944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,255,0.15240533153216043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,255,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,255,0.13038933277130127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,255,0.14882133404413858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,255,0.13926399747530618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,255,0.1358506679534912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,255,0.130730668703715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,255,0.13260799646377563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,255,0.1302186648050944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,255,0.12987732887268066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,511,0.24968532721201578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,511,0.2542933424313863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,511,0.24900267521540323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,511,0.24627200762430826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,511,0.24541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,511,0.2635093331336975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,511,0.2461013396581014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,511,0.2486613392829895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,511,0.25412267446517944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,511,0.2501973311106364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,511,0.24661332368850708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,511,0.24524799982706705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,511,0.26521599292755127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,511,0.2455893357594808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,1023,0.4928853511810303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,1023,0.48179201285044354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,1023,0.47889065742492676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,1023,0.4775253136952718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,1023,0.47496533393859863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,1023,0.4756480058034261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,1023,0.4747946659723918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,1023,0.48264535268147785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,1023,0.47598934173583984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,1023,0.47496533393859863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,1023,0.4787199894587199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,1023,0.4753066698710124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,1023,0.49664000670115155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,1023,0.4768426815668742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,2047,0.9410560131072998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,2047,0.9335467020670573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,2047,0.9352533022562662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,2047,0.9494187037150065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,2047,0.9349119663238525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,2047,0.93559463818868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,2047,0.9408853054046631
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,2047,0.9350826740264893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,2047,0.9347413380940756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,2047,0.935424009958903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,2047,0.934058666229248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,2047,0.9485653241475424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,2047,0.9332053661346436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,2047,0.9357653458913168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,4095,1.848149299621582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,4095,1.8717013994852703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,4095,1.8520746231079102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,4095,1.8513919512430828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,4095,1.8501973152160645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,4095,1.8512214024861653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,4095,1.8491733868916829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,4095,1.8508799870808919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,4095,1.8682880401611328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,4095,1.8500266075134277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,4095,1.8539519309997559
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,4095,1.8510506947835286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,4095,1.849514643351237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,4095,1.8512214024861653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,8191,3.6853758494059243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,8191,3.6913493474324546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,8191,3.7019306818644204
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,8191,3.6904961268107095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,8191,3.6944214502970376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,8191,3.7143894831339517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,8191,3.6918614705403647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,8191,3.6881065368652344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,8191,3.6886186599731445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,8191,3.6927146911621094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,8191,3.7005653381347656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,8191,3.71507199605306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,8191,3.688277244567871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,8191,3.698176066080729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,16383,7.41102917989095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,16383,7.363925298055013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,16383,7.366997400919597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,16383,7.371775945027669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,16383,7.399765650431315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,16383,7.374165217081706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,16383,7.378602981567383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,16383,7.439701080322266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,16383,7.36187744140625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,16383,7.372458775838216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,16383,7.378773371378581
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,16383,7.376213073730469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,16383,7.3920853932698565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,16383,7.379285176595052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,32767,14.79526392618815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,32767,14.732799530029297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,32767,14.747477213541666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,32767,14.791851043701172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,32767,14.738773345947266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,32767,14.81540298461914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,32767,14.744917551676432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,32767,14.791338602701822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,32767,14.782634735107422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,32767,14.756523132324219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,32767,14.713855743408203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,32767,14.701226552327475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,32767,14.77563730875651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,32767,14.828715006510416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,1,65535,29.568341573079426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,2,65535,29.3746337890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,8,65535,29.47089131673177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,16,65535,29.485738118489582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,4,65535,29.553492228190105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,32,65535,29.4833501180013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,256,1,64,65535,29.55639394124349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,1,65535,29.70299784342448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,1,0.08055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,1,0.09045333663622539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,1,0.07372800012429555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,1,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,1,0.07133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,1,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,1,0.07150933146476746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,1,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,1,0.08038400113582611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,1,0.07526400188604991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,1,0.07321600119272868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,1,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,1,0.07150933146476746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,2,65535,29.443583170572918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,1,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,3,0.09181867043177287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,3,0.07406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,3,0.08123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,3,0.07355733215808868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,3,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,8,65535,29.481129964192707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,3,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,3,0.07133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,4,65535,29.385386149088543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,3,0.09233066439628601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,32,65535,29.594454447428387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,3,0.07458133498827617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,3,0.07321600119272868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,3,0.08123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,3,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,3,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,3,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,16,65535,29.458262125651043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,7,0.07577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,7,0.08140799899895985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,7,0.09335466225941975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,7,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,7,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,7,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,7,0.07150933146476746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,7,0.07577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,7,0.09301333626111348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,7,0.07287466526031494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,7,0.08089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,7,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,7,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,7,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,15,0.0820906658967336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,15,0.09403733412424724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,15,0.07714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,15,0.07372800012429555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,15,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,15,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,15,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,15,0.08157866696516673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,15,0.10257066289583842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,15,0.0766293356815974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,15,0.07133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,256,1,64,65535,29.641215006510418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,15,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,15,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,15,0.09591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,31,0.07543466488520305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,31,0.07867733140786488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,31,0.08277333279450734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,31,0.09847467144330342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,31,0.07202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,31,0.07082666456699371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,31,0.07133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,31,0.08379733562469482
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,31,0.07867733140786488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,31,0.07253333429495494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,31,0.07543466488520305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,31,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,31,0.0993280013402303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,31,0.07048533360163371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,63,0.105813334385554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,63,0.07884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,63,0.07611733178297679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,63,0.07526400188604991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,63,0.08891733487447102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,63,0.08140799899895985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,63,0.07577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,63,0.08089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,63,0.07816533247629802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,63,0.0890880028406779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,63,0.07509333391984303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,63,0.07611733178297679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,63,0.0747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,63,0.10393599669138591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,127,0.13192533453305563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,127,0.1353386640548706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,127,0.14097066720326742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,127,0.12970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,127,0.15291733543078104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,127,0.1288533310095469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,127,0.1281706690788269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,127,0.13260799646377563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,127,0.13550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,127,0.14199466506640115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,127,0.1302186648050944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,127,0.12851199507713318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,127,0.1293653349081675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,255,0.237226665019989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,255,0.23961599667867026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,255,0.2486613392829895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,255,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,255,0.26231465737024945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,127,0.1539413332939148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,255,0.23603200912475586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,255,0.23483733336130777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,255,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,255,0.2481493353843689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,255,0.26316799720128375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,255,0.2397866646448771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,255,0.23654399315516153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,255,0.23517866929372153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,511,0.4541440010070801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,511,0.4561920166015625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,511,0.46301865577697754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,511,0.47411199410756427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,255,0.23569067319234213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,511,0.4505600134531657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,511,0.45021867752075195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,511,0.4495360056559245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,511,0.45585068066914874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,511,0.46199464797973633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,511,0.4771840174992879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,511,0.4503893454869588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,511,0.44919466972351074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,1023,0.8989013036092123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,511,0.4532906611760457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,1023,0.884394645690918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,1023,0.8802986939748129
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,1023,0.8797866503397623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,511,0.4503893454869588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,1023,0.8767146269480387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,1023,0.8744959831237793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,1023,0.874837319056193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,1023,0.8801279862721761
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,1023,0.884224017461141
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,1023,0.8797866503397623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,1023,0.8763733704884847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,1023,0.8983893394470215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,1023,0.8825173377990723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,1023,0.8760320345560709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,2047,1.7314133644104004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,2047,1.7271466255187988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,2047,1.730730692545573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,2047,1.7290239334106445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,2047,1.7476266225179036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,2047,1.7264639536539714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,2047,1.7274880409240723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,2047,1.734997272491455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,2047,1.7273173332214355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,2047,1.7273173332214355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,2047,1.7249280611673992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,2047,1.7247573534647624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,2047,1.7500160535176594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,2047,1.726293404897054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,4095,3.44593079884847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,4095,3.4344959259033203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,4095,3.434154510498047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,4095,3.430058797200521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,4095,3.4309120178222656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,4095,3.4315945307413735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,4095,3.4298880894978843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,4095,3.4356905619303384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,4095,3.4247681299845376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,4095,3.436031977335612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,4095,3.4490025838216147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,4095,3.4297173817952475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,4095,3.4322773615519204
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,4095,3.428010622660319
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,8191,6.844586690266927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,8191,6.841856002807617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,8191,6.854314804077148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,8191,6.857215881347656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,8191,6.849194844563802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,8191,6.854314804077148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,8191,6.8449281056722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,8191,6.847829182942708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,8191,6.8444163004557295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,8191,6.846975962320964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,8191,6.857045491536458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,8191,6.845610936482747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,8191,6.853461583455403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,8191,6.847317377726237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,16383,13.70999526977539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,16383,13.685760498046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,16383,13.62841542561849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,16383,13.690879821777344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,16383,13.747370402018229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,16383,13.711530049641928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,16383,13.703679402669271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,16383,13.697364807128906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,16383,13.640021006266275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,16383,13.687466939290365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,16383,13.665791829427084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,16383,13.748736063639322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,16383,13.724671681722006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,16383,13.678421020507812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,2,32767,27.2356694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,1,32767,27.333460489908855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,8,32767,27.42442576090495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,16,32767,27.396779378255207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,32,32767,27.48876698811849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,1,32767,27.332608540852863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,4,32767,27.28533426920573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,512,1,64,32767,27.46777598063151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,1,0.16588800152142844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,1,0.1479680041472117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,1,0.14062933127085367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,1,0.13738666971524557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,1,0.13294933239618936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,1,0.13294933239618936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,1,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,1,0.16622933745384216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,1,0.1469439963499705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,1,0.13755733768145242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,1,0.14199466506640115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,1,0.13294933239618936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,1,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,1,0.13226667046546936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,3,0.16742400328318277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,3,0.14762666821479797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,3,0.14148267110188803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,4,32767,27.322367350260418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,3,0.13772799571355185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,8,32767,27.42493947347005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,3,0.13294933239618936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,16,32767,27.384320576985676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,3,0.13260799646377563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,2,32767,27.238230387369793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,3,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,3,0.14830933014551798
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,3,0.16793600718180338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,3,0.14062933127085367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,3,0.1378986636797587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,3,0.1327786644299825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,3,0.13260799646377563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,3,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,7,0.14199466506640115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,7,0.1384106675783793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,7,0.1474560002485911
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,7,0.169813334941864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,7,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,7,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,7,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,7,0.1479680041472117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,7,0.16895999511082968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,7,0.14216533303260803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,7,0.13329066832860312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,7,0.1389226714769999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,7,0.13329066832860312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,7,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,15,0.1358506679534912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,15,0.16179200013478598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,15,0.1423360009988149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,15,0.1397760013739268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,15,0.17254400253295898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,15,0.1346560021241506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,512,1,64,32767,27.374593098958332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,32,32767,27.490646362304688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,15,0.13380266229311624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,15,0.14899200201034546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,15,0.13567999998728433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,15,0.13943466544151306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,15,0.1430186629295349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,15,0.13482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,15,0.1723733345667521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,15,0.159061332543691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,31,0.15240533153216043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,31,0.1431893308957418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,31,0.14079999923706055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,31,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,31,0.1360213359196981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,31,0.17937066157658896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,31,0.13636266191800436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,31,0.14353066682815552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,31,0.14062933127085367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,31,0.13550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,31,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,31,0.1365333298842112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,31,0.17937066157658896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,31,0.15223466356595358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,63,0.1493333379427592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,63,0.15923200050989786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,63,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,63,0.14421332875887552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,63,0.19541333119074503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,63,0.1389226714769999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,63,0.13858133554458618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,63,0.14882133404413858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,63,0.13994666934013367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,63,0.19524266322453818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,63,0.16025599837303162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,63,0.14455466469128928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,63,0.1384106675783793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,63,0.13926399747530618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,127,0.25548799832661945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,127,0.24490666389465332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,127,0.29047467311223346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,127,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,127,0.23995733261108398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,127,0.2657279968261719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,127,0.24985599517822266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,127,0.2553173303604126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,127,0.24490666389465332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,127,0.2525866627693176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,127,0.24490666389465332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,127,0.26641066869099933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,127,0.29149866104125977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,127,0.244053324063619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,255,0.4522666533788045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,255,0.45823999245961505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,255,0.47189335028330487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,255,0.4466346502304077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,255,0.5061973333358765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,255,0.45311999320983887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,255,0.4442453384399414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,255,0.45687464872996014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,255,0.48076800505320233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,255,0.44680531819661456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,255,0.47189335028330487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,255,0.4991999864578247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,255,0.44287999471028644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,255,0.4510720173517863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,511,0.8659626642862955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,511,0.8753493626912435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,511,0.8521386782328287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,511,0.8509439627329508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,511,0.859818696975708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,511,0.904362678527832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,511,0.8715946674346924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,511,0.9035092989603678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,511,0.8644266923268636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,511,0.8773972988128662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,511,0.85316268603007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,511,0.8802986939748129
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,511,0.85316268603007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,511,0.8500906626383463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,1023,1.7184425989786785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,1023,1.675264040629069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,1023,1.6757760047912598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,1023,1.6680960655212402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,1023,1.6653653780619304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,1023,1.6687787373860676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,1023,1.6839680671691895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,1023,1.675264040629069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,1023,1.6684373219807942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,1023,1.6759467124938965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,1023,1.6651946703592937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,1023,1.6677546501159668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,1023,1.6839680671691895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,1023,1.718272050221761
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,2047,3.3041067123413086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,2047,3.295914649963379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,2047,3.299157460530599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,2047,3.3006931940714517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,2047,3.2916479110717773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,2047,3.2962560653686523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,2047,3.3423360188802085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,2047,3.2892586390177407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,2047,3.2983039220174155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,2047,3.2938667933146157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,2047,3.2909653981526694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,2047,3.31110413869222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,2047,3.3414827982584634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,2047,3.2967678705851235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,4095,6.594218571980794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,4095,6.545408248901367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,4095,6.542336146036784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,4095,6.548138936360677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,4095,6.565717061360677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,4095,6.534997304280599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,4095,6.548479715983073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,4095,6.603946685791016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,4095,6.543018976847331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,4095,6.5425065358479815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,4095,6.54745610555013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,4095,6.5378983815511065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,4095,6.560426712036133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,4095,6.5448958079020185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,8191,13.055658976236979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,8191,13.174613952636719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,8191,13.046101888020834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,8191,13.054976145426432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,8191,13.100031534830729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,8191,13.050880432128906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,8191,13.070507049560547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,8191,13.120340983072916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,8191,13.0611203511556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,8191,13.086719512939453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,8191,13.057365417480469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,8191,13.06811777750651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,8191,13.104810078938803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,8191,13.066581726074219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,2,16383,26.02734883626302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,16,16383,26.111488342285156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,1,1024,1,64,16383,26.2283935546875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,32,16383,26.180948893229168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,4,16383,26.140159606933594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,8,16383,26.22515106201172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,1,16383,26.201087951660156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,1,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,1,16383,26.201087951660156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,3,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,3,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,7,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,7,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,7,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,15,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,15,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,15,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,31,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,2,16383,26.03400421142578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,8,16383,26.171221415201824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,63,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,4,16383,26.066431681315105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,63,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,16,16383,26.15125274658203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,63,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,63,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,127,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,255,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,255,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,511,0.06758399804433186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,511,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,511,0.06741333504517873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,511,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,1023,0.1256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,1023,0.12578133742014566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,1023,0.12595199545224509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,1023,0.12595199545224509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,1023,0.12578133742014566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,32,16383,26.20740254720052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,1023,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,2047,0.24302933613459268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,2047,0.2435413400332133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,2047,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,2047,0.24285866816838583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,2047,0.24285866816838583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,2047,0.24251733223597208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,2047,0.24285866816838583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,2047,0.24183466037114462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,2047,0.244053324063619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,4095,0.47786664962768555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,4095,0.47701334953308105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,4095,0.4771840174992879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,4095,0.4766720136006673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,4095,0.4763306776682536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,4095,0.47701334953308105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,4095,0.47786664962768555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,4095,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,4095,0.47650134563446045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,4095,0.4776959816614787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,4095,0.4763306776682536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,4095,0.4776959816614787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,4095,0.47650134563446045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,4095,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,8191,0.9449813365936279
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,8191,0.9468586444854736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,8191,0.9454933007558187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,8191,0.9446400006612142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,8191,0.9446400006612142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,8191,0.9454933007558187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,8191,0.9466880162556967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,8191,0.9475413163503011
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,8191,0.9449813365936279
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,8191,0.9446400006612142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,16383,1.8865493138631184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,16383,1.8834773699442546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,16383,1.8853546778361003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,16383,1.8810879389444988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,16383,1.8821120262145996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,16383,1.8814293543497722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,16383,1.8891092936197917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,16383,1.8853546778361003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,16383,1.8874026934305828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,16383,1.8836480776468914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,16383,1.8817706108093262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,16383,1.8838186264038086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,32767,3.8925654093424478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,16383,1.8812586466471355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,16383,1.8860373497009277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,32767,3.891711870829264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,1,1024,1,64,16383,26.266454060872395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,32767,3.8842026392618814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,32767,3.8821547826131186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,32767,3.88590939839681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,32767,3.7853867212931314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,32767,3.889834721883138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,32767,3.891882578531901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,32767,3.8835201263427734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,32767,3.8830080032348633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,32767,3.7800960540771484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,32767,3.8860801060994468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,32767,4.010496139526367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,32767,3.785728136698405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,65535,7.773354848225911
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,65535,7.768234888712565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,65535,7.758677164713542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,65535,7.761407852172852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,65535,7.757312138875325
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,65535,7.927295684814453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,65535,8.004266738891602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,65535,7.777109146118164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,65535,7.773525238037109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,65535,7.769258499145508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,65535,7.754069646199544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,65535,7.762432098388672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,65535,7.754581451416016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,65535,7.928661346435547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1,1,1,131071,15.535786946614584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1,1,4,131071,15.513258616129557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1,1,2,131071,15.92678451538086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1,1,8,131071,15.515307108561197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1,1,16,131071,15.49465560913086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1,1,64,131071,15.49841054280599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1,1,32,131071,15.514453887939453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1,1,1,131071,15.539199829101562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1,1,2,131071,15.968084971110025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1,1,16,131071,15.508650461832682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1,1,4,131071,15.51308822631836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1,1,8,131071,15.833770751953125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1,1,32,131071,15.494144439697266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1,1,64,131071,15.498240152994791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,1,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,3,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,3,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,3,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,7,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,7,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,7,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,7,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,15,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,15,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,15,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,31,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,31,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,63,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,63,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,127,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,255,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,255,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,255,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,511,0.06758399804433186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,511,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,511,0.0846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,511,0.06758399804433186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,511,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,1023,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,1023,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,1023,0.1264639993508657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,1023,0.12578133742014566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,1023,0.12612266341845194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,1023,0.1256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,1023,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,1023,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,2047,0.24285866816838583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,2047,0.24302933613459268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,2047,0.24251733223597208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,2047,0.24285866816838583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,2047,0.244053324063619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,2047,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,4095,0.47786664962768555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,4095,0.4771840174992879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,4095,0.4766720136006673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,4095,0.4773546854654948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,4095,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,4095,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,4095,0.47837865352630615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,4095,0.4775253136952718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,4095,0.4768426815668742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,4095,0.47701334953308105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,4095,0.4763306776682536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,4095,0.47650134563446045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,8191,0.9454933007558187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,8191,0.9468586444854736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,8191,0.9535146554311117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,4095,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,8191,0.9460053443908691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,8191,0.9446400006612142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,4095,0.4766720136006673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,8191,0.9521493117014567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,8191,0.9449813365936279
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,8191,0.9483946959177653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,8191,0.9458346366882324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,8191,0.9680213133494059
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,8191,0.9451519648234049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,8191,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,16383,1.9491839408874512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,16383,1.9877546628316243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,16383,1.9802452723185222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,16383,1.9840000470479329
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,16383,1.9595947265625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,16383,1.9927040735880535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,16383,1.9563520749409993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,16383,2.0094292958577475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,16383,1.9872426986694336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,16383,1.9848532676696777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,16383,1.9950933456420898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,16383,1.951573371887207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,16383,1.9927040735880535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,16383,1.9845120112101238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,32767,3.9069013595581055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,32767,3.970730781555176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,32767,4.005205472310384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,32767,3.9690240224202475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,32767,3.969365437825521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,32767,3.9864320755004883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,32767,3.901952107747396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,32767,3.8937600453694663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,32767,3.9069013595581055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,32767,3.975338617960612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,32767,3.993429183959961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,32767,3.985408147176107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,32767,3.980799992879232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,65535,7.929344177246094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,65535,7.937365214029948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,32767,3.9645865758260093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,65535,7.802879969278972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,65535,7.801514943440755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,65535,7.93173344930013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,65535,7.79588254292806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,65535,7.960746765136719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,65535,7.944874445597331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,65535,7.973375956217448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,65535,7.930026372273763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,65535,7.776938756306966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,65535,7.773013432820638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,65535,7.934975941975911
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,65535,7.930367787679036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,2,1,2,131071,15.995733896891275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,2,1,8,131071,15.863978068033854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,2,1,4,131071,15.854080200195312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,2,1,1,131071,15.595349629720053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,2,1,1,131071,15.576917012532553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,2,1,64,131071,15.846570332845053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,2,1,32,131071,15.533738454182943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,2,1,16,131071,15.542271931966146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,2,1,2,131071,15.995733896891275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,2,1,4,131071,15.87814458211263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,2,1,8,131071,15.837525685628256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,1,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,2,1,32,131071,15.842987060546875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,2,1,64,131071,15.846912384033203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,2,1,16,131071,15.90783945719401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,1,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,3,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,3,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,7,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,3,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,7,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,7,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,7,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,15,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,15,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,15,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,15,0.03532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,31,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,31,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,63,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,63,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,63,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,127,0.02372266600529353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,127,0.02372266600529353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,127,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,255,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,255,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,255,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,511,0.07048533360163371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,511,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,511,0.06758399804433186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,511,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,511,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,1023,0.1262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,1023,0.1256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,1023,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,1023,0.12578133742014566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,1023,0.12578133742014566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,1023,0.12612266341845194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,1023,0.1256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,2047,0.2435413400332133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,2047,0.24302933613459268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,2047,0.24251733223597208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,2047,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,2047,0.24285866816838583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,2047,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,2047,0.24302933613459268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,4095,0.478549321492513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,2047,0.24320000410079956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,4095,0.4768426815668742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,4095,0.47701334953308105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,4095,0.4766720136006673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,4095,0.4766720136006673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,4095,0.4952746629714966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,4095,0.4763306776682536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,4095,0.47701334953308105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,4095,0.4766720136006673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,4095,0.4775253136952718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,4095,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,4095,0.48622934023539227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,4095,0.47650134563446045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,4095,0.4763306776682536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,8191,0.98798934618632
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,8191,1.0050559838612874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,8191,1.0091520150502522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,8191,1.0042026837666829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,8191,0.9980586369832357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,8191,1.0089813073476155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,8191,1.001471996307373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,8191,1.000106652577718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,8191,1.0084693431854248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,8191,1.0035200119018555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,8191,1.01256529490153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,8191,0.9888426462809244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,16383,1.9653973579406738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,16383,1.999872048695882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,8191,0.9965226650238037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,16383,1.9983359972635906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,16383,2.0037973721822104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,8191,1.0089813073476155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,16383,2.0089173316955566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,16383,2.010965347290039
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,16383,1.9949226379394531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,16383,1.9636905988057454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,16383,2.0003840128580728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,16383,2.002943992614746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,16383,2.0089173316955566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,16383,2.0003840128580728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,16383,2.004821300506592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,32767,3.9210666020711265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,32767,3.9982080459594727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,32767,3.9920641581217446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,32767,4.003669420878093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,32767,4.0038401285807295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,16383,2.0102826754252114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,32767,3.9202133814493814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,32767,3.9741439819335938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,32767,4.017663955688477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,32767,3.9982080459594727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,32767,4.002645174662272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,32767,3.9925759633382163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,32767,4.002474784851074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,32767,4.002133369445801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,32767,3.9741439819335938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,65535,7.9467519124348955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,65535,7.827797571818034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,65535,7.961599985758464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,65535,7.989077250162761
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,65535,8.052223841349283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,65535,7.989077250162761
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,65535,8.00102424621582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,65535,8.003583908081055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,65535,7.833258946736653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,65535,7.9627946217854815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,65535,7.98634656270345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,65535,7.94265619913737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,65535,7.999317169189453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,65535,7.93992551167806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,4,1,4,131071,15.891797383626303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,4,1,1,131071,15.64364751180013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,4,1,2,131071,16.09130605061849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,4,1,8,131071,15.912960052490234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,4,1,16,131071,15.974571228027344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,4,1,1,131071,15.63272476196289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,4,1,64,131071,15.999146779378256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,4,1,32,131071,15.857322692871094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,4,1,2,131071,15.9726931254069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,4,1,4,131071,16.00204849243164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,4,1,16,131071,15.910741170247396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,1,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,4,1,8,131071,16.000340779622395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,4,1,32,131071,15.977471669514975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,4,1,64,131071,15.861930847167969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,1,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,3,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,3,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,3,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,7,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,7,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,7,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,15,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,15,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,15,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,15,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,31,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,31,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,63,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,63,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,127,0.02372266600529353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,255,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,255,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,255,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,255,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,511,0.0679253339767456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,511,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,511,0.06758399804433186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,511,0.06741333504517873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,511,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,1023,0.1262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,1023,0.12595199545224509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,1023,0.1256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,1023,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,1023,0.1262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,1023,0.1256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,1023,0.1262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,2047,0.24251733223597208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,2047,0.24388267596562704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,2047,0.24524799982706705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,2047,0.24320000410079956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,2047,0.24302933613459268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,2047,0.24251733223597208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,2047,0.24849067131678262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,2047,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,2047,0.24200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,2047,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,2047,0.24320000410079956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,4095,0.49937065442403156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,4095,0.5376000006993612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,4095,0.5101226568222046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,4095,0.5090986490249634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,4095,0.5099519888559977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,4095,0.5109759966532389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,4095,0.5094399849573771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,4095,0.5138773520787557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,4095,0.501418670018514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,4095,0.5108053286870321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,4095,0.5102933247884115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,4095,0.5097813208897909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,4095,0.5089279810587565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,8191,1.0091520150502522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,8191,1.0117119948069255
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,8191,1.0168320337931316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,8191,1.0098346869150798
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,4095,0.5106346607208252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,8191,1.0139306386311848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,8191,1.0100053151448567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,8191,1.0106879870096843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,8191,1.0122240384419758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,8191,1.0359466870625813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,8191,1.0106879870096843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,8191,1.0120533307393391
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,8191,1.0120533307393391
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,8191,1.0134186744689941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,8191,1.011029322942098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,16383,2.010965347290039
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,16383,2.009770711263021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,16383,2.008234659830729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,16383,2.0113066037495932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,16383,2.0278612772623696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,16383,2.014890670776367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,16383,2.0196693738301597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,16383,2.020352045694987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,16383,1.9672746658325195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,16383,2.010453383127848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,16383,2.0101119677225747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,16383,2.0188159942626953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,16383,2.0152319272359214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,32767,3.924821217854818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,32767,4.035413424173991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,16383,2.016767978668213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,32767,4.019370714823405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,32767,4.01800537109375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,32767,4.015103975931804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,32767,4.032341321309407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,32767,4.024661382039388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,32767,3.924309412638346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,32767,4.016640027364095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,32767,4.016640027364095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,32767,4.034559885660808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,32767,4.015445391337077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,32767,4.034389177958171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,32767,4.024661382039388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,65535,8.003413518269857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,65535,8.079018910725912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,65535,8.030719757080078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,65535,8.027648289998373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,65535,8.020821253458658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,65535,8.078336079915365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,65535,8.013482411702475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,65535,8.046762466430664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,65535,8.044373194376627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,65535,7.841450373331706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,65535,8.027648289998373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,65535,8.026624043782553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,65535,8.04147211710612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,65535,8.022186915079752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,8,1,2,131071,16.139264424641926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,8,1,1,131071,15.69484837849935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,8,1,4,131071,16.093013763427734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,8,1,16,131071,16.055296579996746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,8,1,32,131071,16.047274271647137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,8,1,8,131071,16.05734380086263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,8,1,64,131071,16.076629638671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,8,1,1,131071,15.64910888671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,8,1,2,131071,16.138751983642578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,8,1,4,131071,16.0360107421875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,8,1,16,131071,16.054271697998047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,8,1,8,131071,16.087552388509113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,1,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,8,1,32,131071,16.04659144083659
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,8,1,64,131071,16.05563735961914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,1,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,3,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,3,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,3,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,3,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,3,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,7,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,7,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,7,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,7,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,15,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,15,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,15,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,15,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,31,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,31,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,31,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,63,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,63,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,63,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,63,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,63,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,127,0.02372266600529353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,127,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,127,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,127,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,255,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,255,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,255,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,255,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,511,0.06775466601053874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,511,0.06775466601053874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,511,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,511,0.0679253339767456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,1023,0.12595199545224509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,511,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,1023,0.13260799646377563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,1023,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,1023,0.12834133704503378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,1023,0.12612266341845194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,1023,0.12578133742014566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,1023,0.13124266266822815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,1023,0.1256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,1023,0.12731732924779257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,1023,0.12595199545224509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,1023,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,2047,0.2624853253364563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,1023,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,2047,0.2577066620190938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,2047,0.2604373296101888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,2047,0.25941334168116253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,2047,0.2595840096473694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,2047,0.25924267371495563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,2047,0.25753599405288696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,2047,0.25890133778254193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,2047,0.26026666164398193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,2047,0.2600959936777751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,2047,0.25924267371495563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,2047,0.25975465774536133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,2047,0.2624853253364563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,2047,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,4095,0.5032960176467896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,4095,0.5152426560719808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,4095,0.5130240122477213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,4095,0.5130240122477213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,4095,0.5120000044504801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,4095,0.5108053286870321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,4095,0.5155839920043945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,4095,0.504319985707601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,4095,0.5123413403828939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,4095,0.5113173325856527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,4095,0.5123413403828939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,4095,0.5114880005518595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,4095,0.5128533442815145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,8191,0.9937919775644938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,4095,0.5109759966532389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,8191,1.017685333887736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,8191,1.0226346651713054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,8191,1.0280959606170654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,8191,1.0127360026041667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,8191,1.0139306386311848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,8191,1.0164906978607178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,8191,0.9931093056996664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,8191,1.020416021347046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,8191,1.017514705657959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,8191,1.0154666900634766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,8191,1.0122240384419758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,8191,1.0130773385365803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,8191,1.0122240384419758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,16383,2.02513058980306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,16383,2.016256014506022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,16383,2.023082733154297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,16383,2.0147199630737305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,16383,2.015061378479004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,16383,2.0210347175598145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,16383,2.0183040301005044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,16383,1.9770026206970215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,16383,2.0253012975056968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,16383,2.0225706100463867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,16383,2.0174506505330405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,16383,2.0145492553710938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,16383,2.0200106302897134
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,32767,4.028757413228353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,16383,2.023594697316488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,32767,4.044287999471028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,32767,4.034559885660808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,32767,4.026879946390788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,32767,4.02295462290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,32767,4.023978551228841
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,32767,4.028586705525716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,32767,4.057770729064941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,32767,4.0586239496866865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,32767,4.0528214772542315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,32767,4.034901301066081
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,32767,4.0226132074991865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,32767,4.024832089742024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,65535,8.042325337727865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,65535,8.091818491617838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,32767,4.024832089742024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,65535,8.059221267700195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,65535,8.041983922322592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,65535,8.038229624430338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,65535,8.04147211710612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,65535,8.064341227213541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,65535,8.042154947916666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,65535,8.058879852294922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,65535,8.09335454305013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,65535,8.039765040079752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,65535,8.044544219970703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,65535,8.04317855834961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,65535,8.033621470133463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,16,1,2,131071,16.192853291829426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,16,1,1,131071,16.048810323079426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,16,1,4,131071,16.113834381103516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,16,1,8,131071,16.089258829752605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,16,1,16,131071,16.075604756673176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,16,1,32,131071,16.083967844645183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,16,1,64,131071,16.081920623779297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,16,1,1,131071,15.743999481201172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,1,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,16,1,2,131071,16.18824513753255
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,16,1,8,131071,16.10478973388672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,1,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,16,1,4,131071,16.12458674112956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,16,1,32,131071,16.073898315429688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,16,1,16,131071,16.085163116455078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,16,1,64,131071,16.081578572591145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,1,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,1,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,3,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,3,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,3,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,3,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,3,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,3,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,7,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,7,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,7,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,7,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,15,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,15,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,15,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,15,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,31,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,31,0.049322664737701416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,31,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,31,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,63,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,63,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,63,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,63,0.830293337504069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,63,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,127,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,127,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,255,0.05034666756788889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,255,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,255,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,255,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,511,0.09523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,511,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,511,0.0679253339767456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,511,0.06997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,511,0.09523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,511,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,511,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,511,0.06775466601053874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,1023,0.1790293256441752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,1023,0.13806933164596558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,1023,0.14045866330464682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,1023,0.1367039978504181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,1023,0.13619200388590494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,1023,0.1353386640548706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,1023,0.1358506679534912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,1023,0.17885865767796835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,1023,0.13943466544151306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,1023,0.1367039978504181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,1023,0.13550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,1023,0.1641813317934672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,1023,0.13806933164596558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,1023,0.1367039978504181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,2047,0.34508800506591797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,2047,0.2662400007247925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,2047,0.26368000109990436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,2047,0.2616320053736369
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,2047,0.26368000109990436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,2047,0.2624853253364563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,2047,0.2624853253364563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,2047,0.26265599330266315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,2047,0.26658133665720624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,2047,0.34594134489695233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,2047,0.2640213370323181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,2047,0.2624853253364563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,2047,0.2616320053736369
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,4095,0.6775466601053873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,2047,0.26180267333984375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,4095,0.5213866631189982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,4095,0.517632007598877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,4095,0.5145599842071533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,4095,0.5157546599706014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,4095,0.5147306521733602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,4095,0.5150719881057739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,4095,0.6778879960378011
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,4095,0.5155839920043945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,4095,0.5171200037002563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,4095,0.5224106709162394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,4095,0.5150719881057739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,4095,0.5142186482747396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,4095,0.5154133240381876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,8191,1.3414400418599446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,8191,1.0292906761169434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,8191,1.0240000089009602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,8191,1.020586649576823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,8191,1.022976001103719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,8191,1.0190506776173909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,8191,1.0199039777119954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,8191,1.0193920135498047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,8191,1.0308266480763753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,8191,1.021440029144287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,8191,1.0240000089009602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,8191,1.3436586062113445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,8191,1.0193920135498047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,8191,1.0222933292388916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,16383,2.6721280415852866
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,16383,2.038954734802246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,16383,2.044586658477783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,16383,2.0324692726135254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,16383,2.0329813957214355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,16383,2.027520020802816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,16383,2.036224047342936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,16383,2.039466698964437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,16383,2.0473173459370932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,16383,2.6649600664774575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,16383,2.0319573084513345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,16383,2.0329813957214355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,16383,2.0316160519917807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,16383,2.027008056640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,32767,4.087637265523274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,32767,4.056576093037923
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,32767,5.328213373819987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,32767,4.0632321039835615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,32767,4.049578666687012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,32767,4.054357210795085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,32767,4.085077285766602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,32767,4.04855473836263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,32767,4.057429313659668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,32767,5.323605219523112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,32767,4.05845324198405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,32767,4.047871907552083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,32767,4.054357210795085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,32767,4.061183929443359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,65535,8.108885447184244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,65535,10.62178103129069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,65535,8.141482671101889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,65535,8.094378789265951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,65535,8.095232009887695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,65535,8.112469355265299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,65535,8.101717631022135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,65535,10.610687891642252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,65535,8.114517211914062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,65535,8.148138682047525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,65535,8.093184153238932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,65535,8.09062385559082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,65535,8.093866984049479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,65535,8.10530153910319
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,32,1,1,131071,21.235029856363933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,32,1,4,131071,16.21384557088216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,32,1,16,131071,16.178688049316406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,32,1,8,131071,16.184319814046223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,32,1,64,131071,16.189781188964844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,32,1,32,131071,16.16349919637044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,32,1,2,131071,16.294058481852215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,32,1,1,131071,21.253461201985676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,32,1,2,131071,16.363178253173828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,1,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,32,1,4,131071,16.216064453125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,32,1,8,131071,16.175615946451824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,32,1,16,131071,16.18176015218099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,1,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,32,1,32,131071,16.18517303466797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,1,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,1,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,32,1,64,131071,16.165205637613933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,1,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,1,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,1,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,1,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,1,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,3,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,1,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,3,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,3,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,3,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,3,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,3,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,3,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,3,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,7,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,7,0.03293866664171219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,7,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,7,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,7,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,7,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,7,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,7,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,7,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,7,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,7,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,7,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,7,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,7,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,15,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,15,0.031744000812371574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,15,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,15,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,15,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,15,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,15,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,15,0.03259733319282532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,15,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,15,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,15,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,15,0.040106666584809623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,15,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,31,0.03293866664171219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,31,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,31,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,31,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,31,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,31,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,31,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,31,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,31,0.031914666295051575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,31,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,31,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,31,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,31,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,63,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,31,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,63,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,63,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,63,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,63,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,63,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,63,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,63,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,63,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,63,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,63,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,63,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,63,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,127,0.05085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,63,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,127,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,127,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,127,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,127,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,127,0.053077335158983864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,127,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,127,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,127,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,127,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,127,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,127,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,127,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,127,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,255,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,255,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,255,0.0885759989420573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,255,0.05358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,255,0.051370665431022644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,255,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,255,0.08874666690826416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,255,0.050517335534095764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,255,0.05341866612434387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,255,0.05120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,255,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,255,0.050517335534095764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,511,0.16145066420237222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,255,0.0506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,511,0.09642666578292847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,255,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,511,0.0962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,511,0.09557333588600159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,511,0.09608532985051473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,511,0.09796266754468282
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,511,0.09540266791979472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,511,0.09745066364606221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,511,0.16196266810099283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,511,0.09557333588600159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,511,0.09642666578292847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,1023,0.306005338827769
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,511,0.09591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,511,0.09574400385220845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,1023,0.18244266510009766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,511,0.09506133198738098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,1023,0.18090667327245077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,1023,0.1812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,1023,0.18090667327245077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,1023,0.1807360053062439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,1023,0.18158932526906332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,1023,0.18175999323527017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,1023,0.307370662689209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,1023,0.18141865730285645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,1023,0.18141865730285645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,1023,0.18090667327245077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,1023,0.18141865730285645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,1023,0.1807360053062439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,2047,0.5925546487172445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,2047,0.35072000821431476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,2047,0.3519146839777629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,2047,0.3519146839777629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,2047,0.3515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,2047,0.3520853519439697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,2047,0.3519146839777629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,2047,0.3510613441467285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,2047,0.5906773408253988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,2047,0.3515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,2047,0.3515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,2047,0.3515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,2047,0.35225598017374676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,4095,0.6879573663075765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,4095,1.1612160205841064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,4095,0.6913706461588541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,4095,0.6934186617533366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,2047,0.35225598017374676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,4095,0.6898346741994222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,4095,0.6930773258209229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,4095,0.6952959696451823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,4095,0.6908586819966634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,4095,0.6881279945373535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,4095,1.1617279847462971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,4095,0.6922240257263184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,4095,0.7203839619954427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,4095,0.6935893694559733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,4095,0.6941013336181641
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,8191,1.3614080746968586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,8191,1.3648212750752766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,8191,1.3747199376424153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,8191,1.3709653218587239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,8191,2.3174826304117837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,8191,1.3752320607503254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,8191,1.3764266967773438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,8191,1.3607254028320312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,8191,1.36516269048055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,8191,1.3714772860209148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,8191,1.374037265777588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,8191,1.3757440249125164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,8191,2.313898722330729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,8191,1.3784747123718262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,16383,4.615850766499837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,16383,2.7042134602864585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,16383,2.7173547744750977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,16383,2.729130744934082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,16383,2.7354453404744468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,16383,2.739199956258138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,16383,2.7453438440958657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,16383,4.615338643391927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,16383,2.703189214070638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,16383,2.717695871988932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,16383,2.7374931971232095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,16383,2.7499519983927407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,16383,2.727253278096517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,16383,2.7402238845825195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,32767,5.389141082763672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,32767,9.228970845540365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,32767,5.439317067464192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,32767,5.415082931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,32767,5.460479736328125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,32767,5.465087890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,32767,5.467477162679036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,32767,5.389482498168945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,32767,9.185792287190756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,32767,5.414570490519206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,32767,5.472085316975911
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,32767,5.457408269246419
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,32767,5.451093037923177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,32767,5.4695250193278
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,65535,10.74722162882487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,65535,18.344788869222004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,65535,10.816511789957682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,65535,10.862932840983072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,65535,10.909695943196615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,65535,10.925226847330729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,65535,10.942293802897135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,65535,10.757802327473959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,65535,10.815146128336588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,65535,18.390869140625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,65535,10.866517384847006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,65535,10.927103678385416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,65535,10.898090362548828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,65535,10.930346171061197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,64,1,2,131071,21.4652582804362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,64,1,4,131071,21.6101557413737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,64,1,8,131071,21.715627034505207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,64,1,32,131071,21.837483723958332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,64,1,16,131071,21.79157257080078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,64,1,64,131071,21.83611806233724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,64,1,1,131071,36.89386749267578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,1,0.055125330885251365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,64,1,1,131071,36.816724141438804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,64,1,2,131071,21.497515360514324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,64,1,4,131071,21.61322784423828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,1,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,64,1,16,131071,21.79925282796224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,64,1,32,131071,21.82604726155599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,1,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,1,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,1,0.031914666295051575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,64,1,8,131071,21.729451497395832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,64,1,64,131071,21.837140401204426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,1,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,1,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,1,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,1,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,1,0.031914666295051575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,1,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,1,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,3,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,3,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,3,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,3,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,3,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,3,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,3,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,3,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,3,0.031744000812371574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,3,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,3,0.031744000812371574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,3,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,3,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,3,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,7,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,7,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,7,0.031744000812371574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,7,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,7,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,7,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,7,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,7,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,7,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,7,0.05563733478387197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,7,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,7,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,7,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,7,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,15,0.031914666295051575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,15,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,15,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,15,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,15,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,15,0.05580799778302511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,15,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,15,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,15,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,15,0.032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,15,0.05580799778302511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,15,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,15,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,31,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,31,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,31,0.056320001681645714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,31,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,15,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,31,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,31,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,31,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,31,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,31,0.05580799778302511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,31,0.031914666295051575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,31,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,31,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,63,0.05819733440876007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,63,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,31,0.032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,63,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,63,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,31,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,63,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,63,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,63,0.0580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,63,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,63,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,63,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,63,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,63,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,63,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,127,0.05649066468079885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,127,0.04863999783992767
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,127,0.08567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,63,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,127,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,127,0.04471466441949209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,127,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,127,0.04471466441949209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,127,0.08686932921409607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,127,0.048298666874567665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,127,0.057002668579419456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,127,0.045226668318112694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,127,0.04454400142033895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,255,0.09796266754468282
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,127,0.045567999283472695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,127,0.04471466441949209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,255,0.08840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,255,0.14250666896502176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,255,0.08635733524958293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,255,0.08550399541854858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,255,0.08447999755541484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,255,0.08447999755541484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,255,0.10069333513577779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,255,0.14591999848683676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,255,0.08567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,255,0.08482133348782857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,255,0.08823466300964355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,255,0.08635733524958293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,255,0.08413867155710857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,511,0.16247466206550598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,511,0.252074658870697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,511,0.17169066270192465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,511,0.16008533040682474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,511,0.15940266847610474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,511,0.15871999661127725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,511,0.1687893271446228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,511,0.1609386702378591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,511,0.1646933356920878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,511,0.16264533003171286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,511,0.159061332543691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,511,0.25190399090449017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,511,0.15786666671435037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,511,0.1585493286450704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,1023,0.3155626654624939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,1023,0.47786664962768555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,1023,0.3063466747601827
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,1023,0.30907734235127765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,1023,0.30498133103052777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,1023,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,1023,0.47786664962768555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,1023,0.3037866751352946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,1023,0.3160746693611145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,1023,0.3065173427263896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,1023,0.30446932713190716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,1023,0.30907734235127765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,2047,0.9251840114593506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,1023,0.3058346708615621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,1023,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,2047,0.5995519955952963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,2047,0.6058666706085205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,2047,0.6137173175811768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,2047,0.5956266721089681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,2047,0.5952853361765543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,2047,0.5993813276290894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,2047,0.5954560041427612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,2047,0.5956266721089681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,2047,0.6062080065409342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,2047,0.9268906911214193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,2047,0.5951146682103475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,2047,0.596992015838623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,2047,0.5949440002441406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,4095,1.18886399269104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,4095,1.8228905995686848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,4095,1.1779413223266602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,4095,1.1765759785970051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,4095,1.1775999863942463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,4095,1.177770694096883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,4095,1.1798186302185059
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,4095,1.8228905995686848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,4095,1.1793066660563152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,4095,1.1779413223266602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,4095,1.1762346426645915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,4095,1.1774293581644695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,4095,1.18886399269104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,4095,1.1779413223266602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,8191,2.3500800132751465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,8191,2.3633920351664224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,8191,2.344106674194336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,8191,2.3464959462483725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,8191,3.6285438537597656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,8191,2.346325397491455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,8191,2.361685276031494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,8191,2.3454720179239907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,8191,3.6179625193277993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,8191,2.3466666539510093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,8191,2.34769074122111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,8191,2.3490559260050454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,8191,2.3451306025187173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,8191,2.346325397491455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,16383,7.228586832682292
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,16383,4.7143252690633135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,16383,4.688384056091309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,16383,4.709546724955241
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,16383,4.68940798441569
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,16383,4.694016138712565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,16383,4.688042640686035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,16383,4.718250592549642
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,16383,4.682410558064778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,16383,4.685824076334636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,16383,4.686165491739909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,16383,7.220736185709636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,16383,4.6909440358479815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,16383,4.696917215983073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,32767,9.440597534179688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,32767,9.358677546183268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,32767,14.435157775878906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,32767,9.363114674886068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,32767,9.368234634399414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,32767,9.380010604858398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,32767,9.36567497253418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,32767,9.439744313557943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,32767,9.359359741210938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,32767,14.449493408203125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,32767,9.363797505696615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,32767,9.370965321858725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,32767,9.364821116129557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,32767,9.393834431966146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,65535,18.900992075602215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,65535,28.83856964111328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,65535,18.726741790771484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,65535,18.696020762125652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,65535,18.735445658365887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,65535,18.740907033284504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,65535,18.717525482177734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,65535,28.83106231689453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,65535,18.88682683308919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,65535,18.708479563395183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,65535,18.69158426920573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,65535,18.723840077718098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,65535,18.697728474934895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,65535,18.751658121744793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,128,1,2,131071,37.74788157145182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,128,1,8,131071,37.4654286702474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,128,1,4,131071,37.458943684895836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,128,1,16,131071,37.4294179280599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,128,1,32,131071,37.458091735839844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,128,1,1,131071,57.90361531575521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,128,1,64,131071,37.43590291341146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,1,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,1,0.055125330885251365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,1,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,128,1,1,131071,57.853271484375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,1,0.0795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,1,0.04539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,1,0.04437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,128,1,2,131071,37.82075754801432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,1,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,1,0.08994133273760478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,128,1,4,131071,37.41952006022135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,1,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,128,1,8,131071,37.46798960367838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,1,0.05478399991989136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,1,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,1,0.045226668318112694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,128,1,64,131071,37.43658701578776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,1,0.04420266548792521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,128,1,16,131071,37.429761250813804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,128,1,32,131071,37.442901611328125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,1,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,3,0.04727466901143392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,3,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,3,0.045567999283472695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,3,0.09045333663622539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,3,0.04539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,3,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,3,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,3,0.045909335215886436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,3,0.055125330885251365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,3,0.04778666794300079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,3,0.09096533060073853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,3,0.04420266548792521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,3,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,3,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,7,0.05563733478387197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,7,0.04539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,7,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,7,0.04693333307902018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,7,0.09096533060073853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,7,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,7,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,7,0.09113599856694539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,7,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,7,0.04727466901143392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,7,0.0554666668176651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,7,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,7,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,7,0.04386133452256521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,15,0.04761599997679392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,15,0.056661332647005715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,15,0.09233066439628601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,15,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,15,0.045226668318112694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,15,0.04386133452256521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,15,0.045567999283472695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,15,0.04778666794300079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,15,0.05649066468079885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,15,0.09250133236249287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,15,0.045567999283472695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,15,0.04488533238569895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,15,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,15,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,31,0.045226668318112694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,31,0.04795733094215393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,31,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,31,0.04471466441949209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,31,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,31,0.05836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,31,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,31,0.04744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,31,0.045567999283472695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,31,0.05819733440876007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,31,0.09489066402117412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,31,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,31,0.04488533238569895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,63,0.04642133414745331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,63,0.05205333232879639
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,31,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,63,0.09915733337402344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,63,0.04334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,63,0.06195199986298879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,63,0.04471466441949209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,63,0.043178667624791466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,63,0.09898666540781657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,63,0.061610668897628784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,63,0.045909335215886436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,63,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,63,0.051882664362589516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,63,0.04334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,63,0.05256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,127,0.09215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,127,0.14711466431617737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,127,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,127,0.0747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,127,0.08004266520341237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,127,0.07389866809050243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,127,0.07253333429495494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,127,0.14779733618100485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,127,0.09318400422732036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,127,0.0769706666469574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,127,0.07560533285140991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,127,0.07321600119272868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,127,0.08072533210118611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,255,0.14284800489743552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,255,0.1346560021241506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,255,0.1327786644299825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,255,0.1327786644299825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,255,0.24849067131678262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,255,0.15411200126012167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,127,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,255,0.13226667046546936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,255,0.24951465924580893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,255,0.13448533415794373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,255,0.13397333025932312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,255,0.13141333063443503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,255,0.1518933375676473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,511,0.26282666126887005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,511,0.2542933424313863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,255,0.130730668703715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,255,0.1493333379427592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,511,0.45687464872996014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,511,0.24900267521540323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,511,0.24593067169189453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,511,0.24627200762430826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,511,0.2542933424313863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,511,0.2629973292350769
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,511,0.2501973311106364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,511,0.25036799907684326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,511,0.24644267559051514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,511,0.4575573205947876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,511,0.25548799832661945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,511,0.24593067169189453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,1023,0.4920320113499959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,1023,0.48230401674906415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,1023,0.8710827032725016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,1023,0.4773546854654948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,1023,0.4790613253911336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,1023,0.47650134563446045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,1023,0.47650134563446045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,1023,0.4773546854654948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,1023,0.4795733292897542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,1023,0.8724479675292969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,1023,0.48264535268147785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,1023,0.49134933948516846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,1023,0.4763306776682536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,1023,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,2047,0.9497600396474203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,2047,0.9379839897155762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,2047,0.9362773100535074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,2047,0.9350826740264893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,2047,0.9359359741210938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,2047,1.702741305033366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,2047,0.9376426537831625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,2047,0.9492479960123698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,2047,0.93559463818868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,2047,0.9338880379994711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,2047,0.9349119663238525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,2047,0.93559463818868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,2047,1.698304017384847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,2047,0.9362773100535074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,4095,1.8640213012695312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,4095,1.8513919512430828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,4095,1.8520746231079102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,4095,3.357525189717611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,4095,1.8780159950256348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,4095,1.85480531056722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,4095,3.365034739176432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,4095,1.8483200073242188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,4095,1.8501973152160645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,4095,1.864192008972168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,4095,1.8508799870808919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,4095,1.8522453308105469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,4095,1.8520746231079102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,4095,1.8539519309997559
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,8191,6.7003733317057295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,8191,3.6881065368652344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,8191,3.6833279927571616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,8191,3.6887893676757812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,8191,3.690154711405436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,8191,3.7205333709716797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,8191,3.689642588297526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,8191,3.6893014907836914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,8191,6.703957239786784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,8191,3.6834987004597983
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,8191,3.7241172790527344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,8191,3.6908372243245444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,8191,3.6918614705403647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,8191,3.6915200551350913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,16383,7.414101282755534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,16383,13.337258656819662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,16383,7.354368209838867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,16383,7.392768224080403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,16383,7.387306849161784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,16383,7.378773371378581
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,16383,7.384746551513672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,16383,7.413418451944987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,16383,7.363413492838542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,16383,7.355050404866536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,16383,7.371605555216472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,16383,7.379456202189128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,16383,7.394304275512695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,16383,13.335210164388021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,32767,26.7509765625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,32767,14.791168212890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,32767,14.697643280029297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,32767,14.77068837483724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,32767,14.725290934244791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,32767,14.77734375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,32767,14.747989654541016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,32767,26.639017740885418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,32767,14.704639434814453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,32767,14.737407684326172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,32767,14.803797403971354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,32767,14.699178059895834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,32767,14.876842498779297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,32767,14.772565205891928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,256,1,4,65535,29.410303751627605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,256,1,8,65535,29.39630889892578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,256,1,16,65535,29.602645874023438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,256,1,32,65535,29.4835205078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,256,1,1,65535,53.27018737792969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,256,1,64,65535,29.561686197916668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,256,1,2,65535,29.64189910888672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,1,0.16332800189654031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,256,1,1,65535,53.388458251953125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,1,0.09079466263453166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,1,0.08106666803359985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,1,0.07423999905586243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,1,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,1,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,1,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,1,0.09113599856694539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,1,0.16554666558901468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,1,0.08089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,1,0.07406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,1,0.07321600119272868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,1,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,256,1,2,65535,29.61407979329427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,1,0.07133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,256,1,4,65535,29.386070251464844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,256,1,16,65535,29.408767700195312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,3,0.16639999548594156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,3,0.09318400422732036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,3,0.08106666803359985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,3,0.07458133498827617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,256,1,8,65535,29.562538146972656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,3,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,3,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,3,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,3,0.16401066382726034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,3,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,3,0.08089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,3,0.09147733449935913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,3,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,3,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,3,0.07355733215808868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,7,0.08072533210118611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,7,0.09335466225941975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,7,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,7,0.16537599762280783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,7,0.07611733178297679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,7,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,7,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,7,0.08157866696516673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,7,0.07458133498827617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,7,0.09215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,256,1,32,65535,29.502293904622395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,7,0.16759467124938965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,7,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,7,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,7,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,15,0.09471999605496724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,15,0.07594666878382365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,15,0.08157866696516673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,15,0.16861865917841592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,15,0.07321600119272868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,15,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,15,0.0817493349313736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,256,1,64,65535,29.594454447428387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,15,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,15,0.09523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,15,0.0773119976123174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,15,0.16913066307703653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,15,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,15,0.07082666456699371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,15,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,31,0.17032533884048462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,31,0.09966933727264404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,31,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,31,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,31,0.07202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,31,0.07901866734027863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,31,0.07867733140786488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,31,0.08294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,31,0.08328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,31,0.0988159974416097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,31,0.07577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,31,0.17151999473571777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,31,0.07133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,63,0.0817493349313736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,63,0.08823466300964355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,63,0.09727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,31,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,63,0.0766293356815974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,63,0.17834667364756265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,63,0.07492266595363617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,63,0.10513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,63,0.08789333701133728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,63,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,63,0.10513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,63,0.0766293356815974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,63,0.07850666840871175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,63,0.07509333391984303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,63,0.08123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,127,0.1360213359196981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,127,0.15479466319084167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,127,0.13192533453305563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,127,0.1423360009988149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,127,0.1288533310095469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,127,0.27426133553187054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,127,0.13567999998728433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,127,0.1295360028743744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,127,0.13294933239618936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,127,0.1534293293952942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,127,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,127,0.12987732887268066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,127,0.27357866366704303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,127,0.12919466694196066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,255,0.24849067131678262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,255,0.4705280065536499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,255,0.23961599667867026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,255,0.24251733223597208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,255,0.23637332518895468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,255,0.23654399315516153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,255,0.26129066944122314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,255,0.26265599330266315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,255,0.24832000335057577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,255,0.24234666426976523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,255,0.23705599705378214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,255,0.23995733261108398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,255,0.4700160026550293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,255,0.23517866929372153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,511,0.46250665187835693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,511,0.475818673769633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,511,0.45585068066914874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,511,0.45448533693949383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,511,0.4505600134531657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,511,0.8669866720835367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,511,0.4621653159459432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,511,0.44970667362213135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,511,0.45585068066914874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,511,0.45397333304087323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,511,0.47547733783721924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,511,0.4505600134531657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,511,0.868010679880778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,511,0.4500480095545451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,1023,0.8792746861775717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,1023,1.6653653780619304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,1023,0.889685312906901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,1023,0.884224017461141
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,1023,0.8811519940694174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,1023,0.899072011311849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,1023,0.8743253548940023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,1023,0.9006079832712809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,1023,0.8850773175557455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,1023,0.8797866503397623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,1023,0.8773972988128662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,1023,0.8773972988128662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,1023,0.8743253548940023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,1023,1.662293275197347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,2047,3.246250788370768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,2047,1.7525760332743328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,2047,1.7329492568969727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,2047,1.731584072113037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,2047,1.7286826769510906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,2047,1.7249280611673992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,2047,1.7252693176269531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,2047,3.244373321533203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,2047,1.7285119692484539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,2047,1.7344853083292644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,2047,1.731584072113037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,2047,1.726293404897054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,2047,1.747114658355713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,2047,1.724415938059489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,4095,3.4263038635253906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,4095,3.433471997578939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,4095,3.4455893834431968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,4095,6.419968287150065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,4095,3.437568028767904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,4095,3.4286934534708657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,4095,3.4303998947143555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,4095,6.408704121907552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,4095,3.4333012898763022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,4095,3.450197219848633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,4095,3.4618028004964194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,4095,3.425450642903646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,4095,3.4310827255249023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,4095,3.430570602416992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,8191,12.768255869547525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,8191,6.838783899943034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,8191,6.857898712158203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,8191,6.838101069132487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,8191,6.855679829915364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,8191,6.848853429158528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,8191,6.846293131510417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,8191,12.780202229817709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,8191,6.864725112915039
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,8191,6.845098495483398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,8191,6.847658793131511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,8191,6.841685612996419
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,8191,6.82803217569987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,8191,6.850730895996094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,16383,25.466880798339844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,16383,13.663744608561197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,16383,13.656747182210287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,16383,13.678421020507812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,16383,13.685760498046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,16383,13.719039916992188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,16383,13.6900266011556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,16383,25.565012613932293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,16383,13.681493123372396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,16383,13.71460215250651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,16383,13.696000417073568
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,16383,13.642922719319662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,16383,13.67739741007487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,16383,13.743104298909506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,512,1,32,32767,27.409749348958332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,512,1,64,32767,27.36896006266276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,512,1,8,32767,27.293866475423176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,512,1,4,32767,27.21484883626302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,512,1,2,32767,27.46606953938802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,512,1,16,32767,27.352747599283855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,512,1,1,32767,50.90389506022135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,1,0.3165866732597351
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,1,0.16605866948763529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,1,0.14728533228238425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,512,1,1,32767,51.00407409667969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,1,0.1418239971001943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,1,0.16110933820406595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,1,0.13636266191800436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,1,0.1327786644299825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,1,0.16639999548594156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,1,0.3170986572901408
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,1,0.14779733618100485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,1,0.14148267110188803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,1,0.13704533378283182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,512,1,2,32767,27.350016276041668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,1,0.13260799646377563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,512,1,4,32767,27.30530039469401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,1,0.13226667046546936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,512,1,8,32767,27.28430938720703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,3,0.14779733618100485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,3,0.14148267110188803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,3,0.16674133141835532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,512,1,16,32767,27.31690724690755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,3,0.3184640010197957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,3,0.13687466581662497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,3,0.13329066832860312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,3,0.1327786644299825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,3,0.14779733618100485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,3,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,3,0.1378986636797587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,3,0.13294933239618936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,3,0.3176106611887614
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,3,0.16742400328318277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,3,0.13226667046546936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,7,0.14165332913398743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,7,0.1693013310432434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,7,0.14813866217931113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,7,0.13858133554458618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,7,0.13380266229311624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,7,0.13209600249926248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,7,0.31692800919214886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,7,0.1418239971001943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,7,0.16827734311421713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,7,0.13858133554458618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,7,0.14830933014551798
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,7,0.13414399822553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,7,0.3193173408508301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,7,0.13192533453305563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,15,0.14165332913398743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,15,0.1358506679534912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,15,0.1397760013739268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,15,0.1723733345667521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,15,0.14967466394106546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,512,1,64,32767,27.357012430826824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,15,0.3211946686108907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,15,0.13482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,15,0.15001599987347922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,15,0.17271467049916586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,15,0.13994666934013367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,15,0.13550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,15,0.14250666896502176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,512,1,32,32767,27.430399576822918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,15,0.13448533415794373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,15,0.320853332678477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,31,0.15308800339698792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,31,0.14387200276056925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,31,0.13619200388590494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,31,0.14062933127085367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,31,0.17971199750900269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,31,0.1367039978504181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,31,0.3269973397254944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,31,0.14404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,31,0.1843199928601583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,31,0.14079999923706055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,31,0.13636266191800436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,31,0.1397760013739268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,31,0.17988266547520956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,63,0.1599146624406179
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,31,0.32443734010060626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,63,0.33740798632303876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,63,0.19438934326171875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,63,0.1389226714769999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,63,0.14455466469128928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,63,0.13858133554458618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,63,0.14916266997655234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,63,0.16025599837303162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,63,0.1495039959748586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,63,0.14404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,63,0.19490132729212442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,63,0.13926399747530618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,63,0.17066667477289835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,63,0.33791999022165936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,127,0.2916693290074666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,127,0.25446399052937824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,127,0.26641066869099933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,127,0.5265066623687744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,127,0.24541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,127,0.24115200837453207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,127,0.25036799907684326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,127,0.25617067019144696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,127,0.2926933368047078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,127,0.2662400007247925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,127,0.24576000372568765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,127,0.244053324063619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,127,0.25036799907684326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,127,0.5283840099970499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,255,0.4747946659723918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,255,0.45892266432444256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,255,0.4522666533788045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,255,0.44390400250752765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,255,0.502784013748169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,255,0.920746644337972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,255,0.44646398226420086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,255,0.45260798931121826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,255,0.45789865652720135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,255,0.47086934248606366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,255,0.500053326288859
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,255,0.9195520083109537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,255,0.4519253174463908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,255,0.4551680088043213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,511,0.8649386564890543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,511,0.8589653174082438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,511,0.874666690826416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,511,0.8535040219624838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,511,0.9033386707305908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,511,0.8521386782328287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,511,1.7059839566548665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,511,0.8755199909210205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,511,0.853333314259847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,511,0.8582826455434164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,511,0.8637440204620361
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,511,0.8511146704355875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,511,0.9040213425954183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,511,1.7082026799519856
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,1023,1.7175893783569336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,1023,1.6757760047912598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,1023,3.276970545450846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,1023,1.6701439221700032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,1023,1.6648532549540203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,1023,1.6843093236287434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,1023,1.6670719782511394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,1023,3.2819201151529946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,1023,1.7198079427083333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,1023,1.6709973017374675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,1023,1.6653653780619304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,1023,1.6843093236287434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,1023,1.6665600140889485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,1023,1.6786773999532063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,2047,6.411434809366862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,2047,3.302570660909017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,2047,3.347285270690918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,2047,3.2892586390177407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,2047,3.2974507013956704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,2047,3.2899414698282876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,2047,3.295232137044271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,2047,6.399829228719075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,2047,3.303936004638672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,2047,3.291989326477051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,2047,3.292501449584961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,2047,3.2880640029907227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,2047,3.3402878443400064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,2047,3.303765296936035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,4095,12.673365275065104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,4095,6.539605458577474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,4095,6.541823705037435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,4095,6.543189366658528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,4095,6.54643185933431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,4095,6.549333572387695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,4095,6.603776295979817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,4095,12.739584604899088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,4095,6.547285079956055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,4095,6.560426712036133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,4095,6.541653315226237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,4095,6.597632090250651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,4095,6.545578638712565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,4095,6.548479715983073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,8191,25.23426055908203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,8191,13.043712615966797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,8191,13.113173166910807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,8191,13.06948216756185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,8191,13.063339233398438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,8191,13.046272277832031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,8191,13.160959879557291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,8191,25.23101806640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,8191,13.03432591756185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,8191,13.1505495707194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,8191,13.058731079101562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,8191,13.110613505045572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,8191,13.057365417480469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,8191,13.066410064697266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,32,1024,1,4,16383,26.02410634358724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,16,1024,1,8,16383,26.12036387125651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,2,1024,1,64,16383,26.141695658365887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,8,1024,1,16,16383,26.08110809326172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,4,1024,1,32,16383,26.251604715983074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,64,1024,1,2,16383,26.179412841796875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,128,1024,1,1,16383,50.2816416422526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,fp8,128,1024,1,1,16383,50.496002197265625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,64,1024,1,2,16383,26.224469502766926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,32,1024,1,4,16383,26.044586181640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,16,1024,1,8,16383,26.057044982910156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,8,1024,1,16,16383,26.07940165201823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,4,1024,1,32,16383,26.11216990152995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_generation,default,float16,float16,2,1024,1,64,16383,26.126848856608074
