framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,1,0.027002667387326557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,1,0.025461333493391674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,1,0.026367999613285065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,1,0.02664533257484436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,1,0.02646933247645696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,1,0.02638400097688039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,1,0.02607999990383784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,1,0.025637333591779072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,1,0.027050666511058807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,1,0.0431573341290156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,1,0.025477332373460133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,1,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,1,0.02571200082699458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,3,0.02678400029738744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,1,0.025706666211287182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,3,0.025983999172846477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,3,0.02610666553179423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,3,0.02534399926662445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,3,0.02606400102376938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,3,0.02717333287000656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,3,0.025466665625572205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,3,0.025381334125995636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,3,0.026176000634829204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,3,0.025258667767047882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,3,0.02589866767326991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,3,0.026165333886941273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,3,0.02629866699377696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,3,0.026133333643277485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,7,0.0273333340883255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,7,0.026074667771657307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,7,0.026687999566396076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,7,0.0260959987839063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,7,0.026047999660174053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,7,0.02868266652027766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,7,0.026122666895389557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,7,0.027376001079877216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,7,0.02605333427588145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,7,0.02584533393383026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,7,0.026874666412671406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,7,0.026341333985328674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,7,0.026159999271233875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,7,0.026122666895389557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,15,0.02657066782315572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,15,0.027061333258946735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,15,0.025781333446502686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,15,0.02571733295917511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,15,0.025685332715511322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,15,0.02587733417749405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,15,0.025600001215934753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,15,0.027274665733178455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,15,0.026127999027570088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,15,0.025616000096003216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,15,0.025487999121348064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,15,0.025631998976071674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,15,0.027274665733178455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,31,0.030746666093667347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,15,0.025733334322770435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,31,0.02677333354949951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,31,0.025306666890780132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,31,0.02532266577084859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,31,0.025279998779296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,31,0.028501334289709728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,31,0.026842666169007618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,31,0.025242666403452556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,31,0.031530665854612984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,31,0.025397333006064098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,31,0.025290665527184803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,31,0.02532800038655599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,31,0.025279998779296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,31,0.02550400048494339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,63,0.027237333357334137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,63,0.025498665869235992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,63,0.024149333437283833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,63,0.024421334266662598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,63,0.02430933217207591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,63,0.024271999796231587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,63,0.025936000049114227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,63,0.033376000821590424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,63,0.02446399877468745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,63,0.024821333587169647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,63,0.024192000428835552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,63,0.024288001159826916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,63,0.024346667031447094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,127,0.027098665634791057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,63,0.024517332514127094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,127,0.02864533414443334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,127,0.026762666801611584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,127,0.02632533262173335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,127,0.026261332134405773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,127,0.026314665873845417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,127,0.026170666019121807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,127,0.027386667827765148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,127,0.02844800055027008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,127,0.026698666314284008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,127,0.026154667139053345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,127,0.026176000634829204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,127,0.026133333643277485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,127,0.026154667139053345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,255,0.028442665934562683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,255,0.027280000348885853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,255,0.026778665681680042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,255,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,255,0.026352000733216602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,255,0.026101333399613697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,255,0.02604266752799352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,255,0.028421332438786823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,255,0.02734400083621343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,255,0.02672533442576726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,255,0.026341333985328674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,255,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,255,0.026074667771657307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,255,0.02619733413060506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,511,0.028954667349656422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,511,0.027903998891512554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,511,0.027242665489514668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,511,0.026954665780067444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,511,0.026975999275843304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,511,0.02665599932273229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,511,0.026672000686327618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,511,0.027994667490323383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,511,0.028864001234372456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,511,0.026869334280490875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,511,0.02720533311367035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,511,0.027290667096773785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,511,0.026789332429567974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,511,0.026634665826956432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,1023,0.030320001145203907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,1023,0.03229333211978277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,1023,0.028922667105992634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,1023,0.027749332288901012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,1023,0.028138667345046997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,1023,0.027989332874615986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,1023,0.0290133332212766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,1023,0.030266667405764263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,1023,0.027808000644048054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,1023,0.027765333652496338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,1023,0.028181334336598713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,1023,0.027786667148272198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,1023,0.027615999182065327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,1023,0.0277813325325648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,2047,0.03363733241955439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,2047,0.03032533327738444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,2047,0.03186666717131933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,2047,0.030832000076770782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,2047,0.030554667115211487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,2047,0.03032533327738444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,2047,0.03044266750415166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,2047,0.03369600077470144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,2047,0.03181333343187968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,2047,0.030559999247392017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,2047,0.030991998811562855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,2047,0.03035733352104823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,2047,0.03049066662788391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,4095,0.038319999972979225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,2047,0.030213333666324615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,4095,0.03467733412981033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,4095,0.03570666660865148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,4095,0.03531199942032496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,4095,0.03461333364248276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,4095,0.034671999514102936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,4095,0.03469866762558619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,4095,0.03811199963092804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,4095,0.034671999514102936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,4095,0.03459733227888743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,4095,0.036176001032193504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,4095,0.03477866699298223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,4095,0.03469866762558619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,4095,0.03465066601832708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,8191,0.039887999494870506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,8191,0.0376800000667572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,8191,0.036837334434191384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,8191,0.03682666768630346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,8191,0.036805334190527596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,8191,0.03669866671164831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,8191,0.036890665690104164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,8191,0.03987200061480204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,8191,0.03777066618204117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,8191,0.036933332681655884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,8191,0.03670933345953623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,8191,0.03685333331425985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,16383,0.05089599887530009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,8191,0.036864000062147774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,8191,0.036805334190527596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,16383,0.047242666284243263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,16383,0.04540266593297323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,16383,0.044682666659355164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,16383,0.044863998889923096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,16383,0.045066664616266884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,16383,0.04465599854787191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,16383,0.0514933317899704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,16383,0.04683200021584829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,16383,0.04544533292452494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,16383,0.04456000030040741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,16383,0.04494933287302653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,16383,0.04489066700140635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,16383,0.04456533491611481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,32767,0.06233599781990051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,32767,0.058415999015172325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,32767,0.05989866455396017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,32767,0.05012799799442291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,32767,0.048672000567118325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,32767,0.04978133241335551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,32767,0.04866666595141093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,32767,0.06206933160622915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,32767,0.05301866432030996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,32767,0.0584746648867925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,32767,0.048810665806134544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,32767,0.04900800188382467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,32767,0.05014933149019877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,32767,0.04822933177153269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,65535,0.07629333436489105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,65535,0.07236800094445546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,65535,0.06986666719118755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,65535,0.06901866694291432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,65535,0.06874133149782817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,65535,0.06810666620731354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,65535,0.0674720009167989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,65535,0.07649066547552745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,65535,0.06990399956703186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,65535,0.07232533395290375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,65535,0.06916800141334534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,65535,0.06857599814732869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,65535,0.06842133402824402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,65535,0.06799466907978058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,8,131071,0.09251200159390767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,1,131071,0.10140800476074219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,4,131071,0.0944640040397644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,2,131071,0.0979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,16,131071,0.09244267145792644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,32,131071,0.09152533610661824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,64,131071,0.09152000149091084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,1,131071,0.10041067004203796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,2,131071,0.09564266602198283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,8,131071,0.09269866347312927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,4,131071,0.09351467092831929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,16,131071,0.0918880005677541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,32,131071,0.09173333644866943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,1,0.02792000025510788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,64,131071,0.09196266531944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,1,0.026975999275843304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,1,0.02646933247645696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,1,0.026346666117509205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,1,0.02641066660483678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,1,0.026335999369621277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,1,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,1,0.028117333849271137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,1,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,1,0.027082666754722595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,1,0.02644266684850057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,1,0.026330667237440746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,1,0.026314665873845417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,3,0.027866666515668232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,1,0.026533332963784535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,3,0.026821332673231762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,3,0.026719999810059864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,3,0.026341333985328674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,3,0.02619733413060506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,3,0.02624000112215678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,3,0.02622933437426885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,3,0.027642667293548584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,3,0.026522666215896606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,3,0.026672000686327618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,3,0.02622399975856145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,3,0.02622399975856145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,3,0.02603733291228612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,3,0.026186667382717133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,7,0.02757333219051361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,7,0.026378666361172993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,7,0.026261332134405773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,7,0.02603733291228612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,7,0.026015999416510265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,7,0.025920001169045765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,7,0.027653334041436512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,7,0.026186667382717133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,7,0.02608533451954524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,7,0.034261333445707955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,7,0.026250667870044708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,7,0.026000000536441803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,7,0.025941332181294758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,15,0.026549334327379864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,15,0.027301333844661713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,7,0.02603200078010559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,15,0.025786665578683216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,15,0.025759999950726826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,15,0.025754667818546295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,15,0.02566933383544286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,15,0.02571733295917511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,15,0.027263998985290527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,15,0.026554666459560394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,15,0.025765334566434223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,15,0.025770666698614757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,15,0.025685332715511322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,15,0.025600001215934753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,15,0.025744001070658367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,31,0.02699733277161916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,31,0.025285333395004272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,31,0.026181332767009735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,31,0.025306666890780132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,31,0.025231999655564625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,31,0.025231999655564625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,31,0.025466665625572205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,31,0.027072000006834667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,31,0.025829332570234936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,31,0.025306666890780132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,31,0.025754667818546295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,31,0.025418666501839954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,31,0.02535466601451238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,31,0.025301332275072735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,63,0.026015999416510265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,63,0.025018667181332905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,63,0.024656000236670177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,63,0.024400000770886738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,63,0.024383999407291412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,63,0.024458666642506916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,63,0.024266667664051056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,63,0.02863999952872594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,63,0.02499733368555705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,63,0.024351999163627625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,63,0.024746666351954143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,63,0.024512000381946564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,63,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,63,0.024432001014550526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,127,0.02881066749493281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,127,0.027237333357334137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,127,0.027029333015282948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,127,0.026821332673231762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,127,0.026341333985328674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,127,0.026250667870044708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,127,0.026346666117509205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,127,0.028837333122889202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,127,0.027210667729377747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,127,0.02700799951950709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,127,0.026522666215896606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,127,0.026352000733216602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,127,0.02629333237806956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,127,0.026426665484905243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,255,0.029178666571776073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,255,0.027215999861558277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,255,0.027509334186712902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,255,0.026591998835404713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,255,0.026682667434215546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,255,0.026447998980681103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,255,0.026560001075267792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,255,0.027690666417280834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,255,0.029253333806991577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,255,0.02699733277161916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,255,0.026687999566396076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,255,0.02657066782315572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,255,0.02649066597223282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,255,0.02651199946800868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,511,0.035786665976047516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,511,0.028602667152881622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,511,0.027808000644048054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,511,0.030058667063713074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,511,0.027615999182065327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,511,0.027653334041436512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,511,0.02738133321205775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,511,0.02718399961789449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,511,0.028757333755493164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,511,0.027888000011444092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,511,0.02762666592995326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,511,0.027589333554108936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,511,0.027482666075229645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,1023,0.03272533416748047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,511,0.027461332579453785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,1023,0.03145066648721695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,1023,0.029989334444204967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,1023,0.029520000020662945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,1023,0.029717333614826202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,1023,0.02957333376010259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,1023,0.029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,1023,0.03134933362404505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,1023,0.029552000264326733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,1023,0.030159999926884968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,1023,0.03292266776164373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,1023,0.029717333614826202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,1023,0.029669334491093952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,1023,0.029359998802344005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,2047,0.036330667634805046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,2047,0.03286933402220408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,2047,0.03342399994532267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,2047,0.03448000053564707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,2047,0.03294933338960012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,2047,0.032698666055997215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,2047,0.03290133426586787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,2047,0.036303999523321785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,2047,0.034688000877698265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,2047,0.03320533285538355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,2047,0.0329120010137558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,2047,0.03262399882078171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,2047,0.0329120010137558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,2047,0.03279466678698858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,4095,0.037978666524092354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,4095,0.0364479993780454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,4095,0.03510399907827377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,4095,0.03480000048875809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,4095,0.034847999612490334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,4095,0.037989333271980286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,4095,0.03494933247566223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,4095,0.036464000741640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,4095,0.034645333886146545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,4095,0.03516799956560135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,4095,0.03474666674931844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,4095,0.035002666215101876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,4095,0.03486400097608566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,4095,0.034847999612490334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,8191,0.040474665661652885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,8191,0.04380266865094503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,8191,0.039733332892258964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,8191,0.039503999054431915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,8191,0.039349332451820374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,8191,0.03944533318281174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,8191,0.0391839991013209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,8191,0.04302933315436045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,8191,0.04075733323891958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,8191,0.03951466580231985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,8191,0.03966933240493139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,8191,0.039434666434923805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,8191,0.03955200066169103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,8191,0.03916800022125244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,16383,0.05435733497142792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,16383,0.058703998724619545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,16383,0.04826666911443075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,16383,0.0452106644709905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,16383,0.04630400240421295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,16383,0.05795200169086456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,16383,0.04572799801826477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,16383,0.044677332043647766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,16383,0.054618666569391884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,16383,0.04876266419887543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,16383,0.04580266773700714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,16383,0.04568000137805939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,16383,0.044821331898371376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,32767,0.06871999800205231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,32767,0.0727040022611618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,16383,0.04625066618124644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,32767,0.06710400183995564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,32767,0.06594666838645935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,32767,0.06529066463311513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,32767,0.07287466526031494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,32767,0.06693333387374878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,32767,0.06573866804440816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,32767,0.06450133522351582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,32767,0.06496533254782359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,32767,0.06545599798361461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,32767,0.06862399975458781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,32767,0.06482666730880737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,32767,0.06462400158246358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,65535,0.09800533453623454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,65535,0.09267733494440715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,65535,0.09027733405431111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,65535,0.08925867080688477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,65535,0.08922132849693298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,65535,0.08871466914812724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,65535,0.09778133034706116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,65535,0.0883893370628357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,65535,0.08931733171145122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,65535,0.0930560032526652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,65535,0.09066133697827657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,65535,0.08899199962615967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,65535,0.08828799923261006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,65535,0.08834667007128398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,4,131071,0.139082670211792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,1,131071,0.14645333091417947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,8,131071,0.13757866621017456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,16,131071,0.13818666338920593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,2,131071,0.14106133580207825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,32,131071,0.13634666800498962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,64,131071,0.13591466347376505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,2,131071,0.1418400009473165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,1,131071,0.14636799693107605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,4,131071,0.13920533657073975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,8,131071,0.1381013294061025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,16,131071,0.13686933120091757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,32,131071,0.13672533631324768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,64,131071,0.1361066699028015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,1,0.028309332827727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,1,0.02917333443959554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,1,0.027034667630990345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,1,0.026714667677879333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,1,0.026677332818508148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,1,0.026378666361172993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,1,0.026693334182103474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,1,0.028304000695546467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,1,0.027269333600997925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,1,0.026949333647886913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,1,0.026602665583292644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,1,0.02666666607062022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,1,0.02641066660483678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,1,0.05949333310127258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,3,0.02701333413521449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,3,0.026965332527955372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,3,0.02805333336194356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,3,0.026421333352724712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,3,0.026474667092164356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,3,0.026378666361172993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,3,0.03088533381621043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,3,0.027749332288901012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,3,0.027434666951497395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,3,0.026880001028378803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,3,0.026543999711672466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,3,0.02646933247645696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,3,0.026517334083716076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,3,0.026447998980681103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,7,0.0276853342851003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,7,0.02717333287000656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,7,0.026752000053723652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,7,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,7,0.026208000878492992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,7,0.026389333109060924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,7,0.02632533262173335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,7,0.026762666801611584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,7,0.027850667635599773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,7,0.027056001126766205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,7,0.026533332963784535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,7,0.02624000112215678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,7,0.02754133443037669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,7,0.02609066665172577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,15,0.02771199991305669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,15,0.026874666412671406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,15,0.026367999613285065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,15,0.0259253333012263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,15,0.025941332181294758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,15,0.025813333690166473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,15,0.026005332668622334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,15,0.027503999571005504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,15,0.026693334182103474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,15,0.026389333109060924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,15,0.025936000049114227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,15,0.02595199892918269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,15,0.025888000925381977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,15,0.02608533451954524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,31,0.02722666660944621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,31,0.02629866699377696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,31,0.026170666019121807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,31,0.025578667720158894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,31,0.025642665723959606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,31,0.025621332228183746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,31,0.027285332481066387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,31,0.025605333348115284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,31,0.026426665484905243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,31,0.02606933315594991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,31,0.025605333348115284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,31,0.025786665578683216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,31,0.025519999365011852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,31,0.02584533393383026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,63,0.02609066665172577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,63,0.02532266577084859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,63,0.02492266645034154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,63,0.024570666253566742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,63,0.037834666669368744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,63,0.024383999407291412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,63,0.024661332368850708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,63,0.02526933451493581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,63,0.02603200078010559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,63,0.025034666061401367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,63,0.02455466737349828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,63,0.024512000381946564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,63,0.024458666642506916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,63,0.024559999505678814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,127,0.0290133332212766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,127,0.028021333118279774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,127,0.027600000301996868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,127,0.026917333404223125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,127,0.027269333600997925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,127,0.026821332673231762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,127,0.026752000053723652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,127,0.029120000700155895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,127,0.02756800005833308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,127,0.028079998989899952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,127,0.027162666122118633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,127,0.026746665438016255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,255,0.02788266787926356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,127,0.02717866748571396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,255,0.029909332593282063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,127,0.02677333354949951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,255,0.02850666642189026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,255,0.027535999814669292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,255,0.027295999228954315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,255,0.02722666660944621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,255,0.0273333340883255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,255,0.028560000161329906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,255,0.02991466720898946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,255,0.0278613343834877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,255,0.027509334186712902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,255,0.027274665733178455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,255,0.027290667096773785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,255,0.027077332139015198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,511,0.03244800120592117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,511,0.03068800022204717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,511,0.029877332349618275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,511,0.02903466671705246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,511,0.02937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,511,0.02884799987077713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,511,0.029178666571776073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,511,0.0323786661028862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,511,0.02924799919128418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,511,0.030671998858451843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,511,0.02914133419593175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,511,0.029904000461101532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,511,0.02903999884923299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,511,0.029194665451844532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,1023,0.035429333647092186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,1023,0.03366933266321818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,1023,0.03241066634654999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,1023,0.03186133255561193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,1023,0.03180799881617228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,1023,0.03161599983771642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,1023,0.0317493329445521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,1023,0.035360001027584076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,1023,0.033626665671666466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,1023,0.03249600032965342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,1023,0.03193599979082743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,1023,0.031727999448776245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,1023,0.039306665460268654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,1023,0.031685332457224526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,2047,0.03770133356253306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,2047,0.035802667339642845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,2047,0.03474666674931844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,2047,0.0340639998515447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,2047,0.033946665624777474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,2047,0.033743999898433685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,2047,0.03379199902216593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,2047,0.03590933233499527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,2047,0.037717332442601524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,2047,0.03481066723664602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,2047,0.038773333032925926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,2047,0.03390933324893316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,2047,0.03409600009520849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,2047,0.033861334125200905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,4095,0.04207466542720795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,4095,0.04016000032424927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,4095,0.038912000755469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,4095,0.04212800165017446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,4095,0.03818666686614355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,4095,0.03782399992148081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,4095,0.03830400109291077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,4095,0.04642133414745331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,4095,0.04011200120051702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,4095,0.03877866764863332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,4095,0.039120001097520195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,4095,0.03812266637881597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,4095,0.03803733239571253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,8191,0.05420800050099691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,4095,0.03819733361403147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,8191,0.0505973349014918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,8191,0.04805333415667216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,8191,0.04332800209522247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,8191,0.04538666705290476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,8191,0.044266665975252785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,8191,0.043882668018341064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,8191,0.0544053316116333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,8191,0.0513919989267985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,8191,0.04799999793370565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,8191,0.04363733530044556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,8191,0.043978666265805565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,8191,0.04391466577847799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,8191,0.043840001026789345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,16383,0.06487999856472015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,16383,0.0665226678053538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,16383,0.07130666573842366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,16383,0.06373333434263866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,16383,0.06332266827424367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,16383,0.06241600215435028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,16383,0.062181333700815834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,16383,0.07181866466999054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,16383,0.06642666459083557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,16383,0.06467199822266896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,16383,0.06347733239332835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,16383,0.06337599953015645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,16383,0.06287466486295064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,16383,0.062319998939832054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,32767,0.09524800380071004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,32767,0.091839998960495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,32767,0.08941866954167683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,32767,0.08748799562454224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,32767,0.08683199683825175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,32767,0.0958720048268636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,32767,0.08646399776140849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,32767,0.08669333656628926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,32767,0.09004799524943034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,32767,0.08746133248011272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,32767,0.08905067046483357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,32767,0.08691199620564778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,32767,0.08665600419044495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,32767,0.08618133266766866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,65535,0.14532267053922018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,65535,0.13993600010871887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,65535,0.1372213363647461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,65535,0.13430933157602945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,65535,0.1351626714070638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,65535,0.13648000359535217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,65535,0.13398399949073792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,65535,0.14591466387112936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,65535,0.1351573367913564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,65535,0.1402133305867513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,65535,0.1368160049120585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,65535,0.13538133104642233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,65535,0.13546666502952576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,65535,0.13387733697891235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,1,131071,0.2415999968846639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,2,131071,0.23568532864252725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,4,131071,0.23250667254130045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,8,131071,0.23029333353042603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,16,131071,0.23217066129048666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,2,131071,0.23518399397532144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,32,131071,0.2301386594772339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,1,131071,0.24198933442433676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,64,131071,0.2295359969139099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,4,131071,0.23297599951426187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,32,131071,0.2303946614265442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,8,131071,0.2302773396174113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,16,131071,0.23210134108861288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,1,0.028538666665554047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,64,131071,0.22909333308537802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,1,0.02787200113137563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,1,0.027647999425729115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,1,0.02733866622050603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,1,0.0272533322374026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,1,0.027141332626342773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,1,0.026922665536403656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,1,0.02861333390076955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,1,0.027552001178264618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,1,0.027637332677841187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,1,0.02717866748571396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,1,0.027232001225153606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,1,0.027056001126766205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,1,0.026911998788515728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,3,0.028533334533373516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,3,0.02740799884001414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,3,0.027450665831565857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,3,0.02714666724205017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,3,0.027024000883102417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,3,0.026746665438016255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,3,0.02701333413521449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,3,0.02861333390076955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,3,0.02752000093460083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,3,0.02741333345572154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,3,0.027098665634791057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,3,0.02701333413521449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,3,0.026762666801611584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,7,0.028399998943010967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,3,0.02697066714366277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,7,0.026837334036827087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,7,0.028746667007605236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,7,0.0272533322374026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,7,0.027669332921504974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,7,0.026778665681680042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,7,0.026949333647886913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,7,0.026719999810059864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,7,0.028512001037597656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,7,0.03401066611210505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,7,0.026954665780067444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,7,0.0269813338915507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,7,0.02678400029738744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,7,0.02696000039577484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,15,0.028207999964555103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,15,0.02664533257484436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,15,0.027317332724730175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,15,0.026672000686327618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,15,0.026447998980681103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,15,0.02632533262173335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,15,0.026608000199000042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,15,0.028234665592511494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,15,0.02658133457104365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,15,0.026954665780067444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,15,0.027141332626342773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,15,0.026629333694775898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,15,0.026357332865397137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,15,0.026426665484905243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,31,0.02754666656255722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,31,0.026799999177455902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,31,0.02678400029738744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,31,0.0262773334980011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,31,0.02630399912595749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,31,0.026250667870044708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,31,0.026741333305835724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,31,0.02595199892918269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,31,0.027514666318893433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,31,0.026848000784715016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,31,0.026015999416510265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,31,0.026261332134405773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,31,0.026133333643277485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,63,0.026848000784715016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,63,0.025439999997615814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,31,0.026234666506449383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,63,0.025701334079106648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,63,0.02496533344189326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,63,0.02516266703605652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,63,0.025050667424996693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,63,0.02513066679239273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,63,0.026709333062171936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,63,0.025573333104451496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,63,0.025050667424996693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,63,0.025040000677108765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,63,0.025279998779296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,63,0.025237334271272022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,63,0.02497066557407379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,127,0.029951999584833782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,127,0.028197333216667175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,127,0.028618666032950085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,127,0.028005334238211315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,127,0.027690666417280834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,127,0.027637332677841187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,127,0.02779199928045273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,127,0.036746665835380554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,127,0.028815999627113342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,127,0.028207999964555103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,127,0.02789866675933202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,127,0.027669332921504974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,127,0.02788266787926356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,127,0.02757866680622101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,255,0.030608000854651134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,255,0.03242133309443792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,255,0.02977599948644638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,255,0.029578665892283123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,255,0.029194665451844532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,255,0.02923733244339625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,255,0.03257066756486893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,255,0.030591999491055805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,255,0.057317331433296204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,255,0.02976000060637792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,255,0.029450667401154835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,255,0.02932800104220708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,255,0.028917332490285236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,255,0.029290666182835896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,511,0.0352906659245491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,511,0.033285332222779594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,511,0.03209066639343897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,511,0.03142933299144109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,511,0.03175999969244003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,511,0.03173333406448364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,511,0.03145066648721695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,511,0.035461333890755974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,511,0.033258666594823204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,511,0.03196266790231069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,511,0.032218667368094124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,511,0.031445334355036415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,511,0.03133866687615713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,511,0.03141866624355316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,1023,0.035631999373435974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,1023,0.03763733307520548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,1023,0.034714666505654655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,1023,0.03440533330043157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,1023,0.0335413341720899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,1023,0.03345066557327906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,1023,0.033530667424201965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,1023,0.034634667138258614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,1023,0.03765333443880081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,1023,0.03469866762558619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,1023,0.03437866767247518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,1023,0.033615998923778534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,1023,0.03345600018898646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,1023,0.03358400116364161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,2047,0.0429013321797053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,2047,0.039647998909155525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,2047,0.038191998998324074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,2047,0.0383146678407987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,2047,0.03762666632731756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,2047,0.0376800000667572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,2047,0.037605332831541695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,2047,0.04301866888999939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,2047,0.039781334499518074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,2047,0.03826666623353958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,2047,0.03825599948565165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,2047,0.03755733370780945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,2047,0.03761066744724909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,2047,0.03770666569471359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,4095,0.054858664671579994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,4095,0.049866666396458946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,4095,0.045466666420300804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,4095,0.04347200194994608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,4095,0.043712000052134194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,4095,0.04331733286380768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,4095,0.042778665820757546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,4095,0.05457599957784017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,4095,0.05091733237107595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,4095,0.04523199796676636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,4095,0.04402133325735728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,4095,0.04338666796684265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,4095,0.04377066592375437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,4095,0.04273599882920583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,8191,0.07186666627724965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,8191,0.06665599842866261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,8191,0.06481599807739258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,8191,0.06380799909432729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,8191,0.062650665640831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,8191,0.07158400118350983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,8191,0.06216000020503998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,8191,0.06228800117969513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,8191,0.06704533100128174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,8191,0.06478933493296306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,8191,0.06343466540177663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,8191,0.06293333570162456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,8191,0.06278400123119354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,8191,0.06221333146095276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,16383,0.096778670946757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,16383,0.09122133255004883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,16383,0.08861333131790161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,16383,0.08669333656628926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,16383,0.08588799834251404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,16383,0.08530132969220479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,16383,0.08582933743794759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,16383,0.09635200103123982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,16383,0.09066667159398396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,16383,0.08835200468699138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,16383,0.08546666304270427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,16383,0.08673600355784099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,16383,0.08589333295822144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,32767,0.13935466607411703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,32767,0.1371999979019165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,32767,0.14531733592351279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,16383,0.08532266815503438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,32767,0.13493333260218301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,32767,0.13449066877365112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,32767,0.13355200489362082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,32767,0.13362133502960205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,32767,0.14707199732462564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,32767,0.13507200280825296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,32767,0.13620799779891968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,32767,0.13919466733932495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,32767,0.13453333576520285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,32767,0.13367467125256857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,32767,0.1334879994392395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,65535,0.23436800638834634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,65535,0.2432373364766439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,65535,0.2316746711730957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,65535,0.22978132963180542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,65535,0.229477326075236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,65535,0.22875199715296426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,65535,0.24162666002909342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,65535,0.22816532850265503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,65535,0.23689067363739014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,65535,0.23149333397547403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,65535,0.22970134019851685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,65535,0.22899732987085977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,65535,0.22990934054056802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,65535,0.22843732436498007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,1,131071,0.42974400520324707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,2,131071,0.4278293450673421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,4,131071,0.42024532953898114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,8,131071,0.41817601521809894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,64,131071,0.4166133403778076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,16,131071,0.4180479844411214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,32,131071,0.4166560173034668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,1,131071,0.42973331610361737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,2,131071,0.4275039831797282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,16,131071,0.4174026648203532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,8,131071,0.41812801361083984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,32,131071,0.41652266184488934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,4,131071,0.41972800095876056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,64,131071,0.4166239897410075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,1,0.030181333422660828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,1,0.028560000161329906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,1,0.029088000456492107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,1,0.028405333558718365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,1,0.02815466622511546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,1,0.028170667588710785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,1,0.028021333118279774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,1,0.028981332977612812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,1,0.030063999195893604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,1,0.028565332293510437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,1,0.028336000939210255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,1,0.031498665610949196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,1,0.028138667345046997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,3,0.029882666965325672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,1,0.028079998989899952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,3,0.02902399996916453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,3,0.028533334533373516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,3,0.02792533238728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,3,0.028016000986099243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,3,0.027999999622503918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,3,0.028031999866167705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,3,0.03669333209594091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,3,0.028773332635561626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,3,0.02826133370399475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,3,0.027935999135176342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,3,0.02796799937884013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,3,0.027984000742435455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,3,0.027978666126728058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,7,0.02992533395687739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,7,0.02805333336194356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,7,0.02870933214823405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,7,0.027813332776228588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,7,0.027765333652496338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,7,0.027903998891512554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,7,0.027717334528764088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,7,0.029733332494894665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,7,0.027813332776228588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,7,0.028704000016053517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,7,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,7,0.02789866675933202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,7,0.027717334528764088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,7,0.027823999524116516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,15,0.02831999957561493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,15,0.02937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,15,0.027829334139823914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,15,0.027642667293548584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,15,0.02754666656255722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,15,0.02739733209212621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,15,0.027664000789324444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,15,0.02951466788848241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,15,0.028399998943010967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,15,0.027823999524116516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,15,0.027615999182065327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,15,0.061162665486335754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,15,0.027424000203609467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,15,0.027658666173617046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,31,0.029103999336560566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,31,0.027957332630952198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,31,0.027450665831565857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,31,0.02703999976317088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,31,0.0271573339899381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,31,0.02719466636578242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,31,0.027258666853109997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,31,0.029066666960716248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,31,0.02794666588306427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,31,0.02759466568628947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,31,0.027274665733178455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,63,0.02794666588306427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,31,0.02718399961789449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,31,0.027034667630990345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,31,0.027237333357334137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,63,0.02693866689999898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,63,0.02659733345111211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,63,0.026122666895389557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,63,0.025978667040665943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,63,0.02587733417749405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,63,0.027930667002995808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,63,0.026144000391165417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,63,0.026965332527955372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,63,0.0266239990790685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,63,0.025936000049114227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,63,0.02586666742960612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,63,0.025962665677070618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,63,0.02595199892918269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,127,0.03314133236805598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,127,0.03130666663249334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,127,0.030048000315825146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,127,0.030202666918436687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,127,0.029850666721661884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,127,0.029637334247430164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,127,0.029696000119050343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,127,0.03306666761636734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,127,0.03129599988460541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,127,0.030266667405764263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,127,0.029904000461101532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,127,0.029904000461101532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,127,0.029669334491093952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,255,0.03481066723664602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,255,0.032858667274316154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,127,0.029701332251230877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,255,0.03209066639343897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,255,0.03197333216667175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,255,0.031717332700888314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,255,0.03182400017976761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,255,0.03169599920511246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,255,0.03214933226505915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,255,0.034714666505654655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,255,0.033002667129039764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,255,0.03188266605138779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,255,0.03169066707293192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,255,0.03170666595300039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,255,0.03197333216667175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,511,0.03806400050719579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,511,0.03482133398453394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,511,0.03608533243338267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,511,0.034416000048319496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,511,0.04423466821511587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,511,0.041589332123597465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,511,0.037978666524092354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,511,0.033701332906881966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,511,0.03542399903138479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,511,0.03405333310365677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,511,0.03367999941110611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,511,0.0377813329299291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,511,0.03417066733042399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,1023,0.04307733476161957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,511,0.03489600121974945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,1023,0.03974399964014689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,1023,0.03872533390919367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,1023,0.038805333276589714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,1023,0.03836799909671148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,1023,0.038533332447210945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,1023,0.039621333281199135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,1023,0.03808533400297165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,1023,0.04312000175317129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,1023,0.03875199953715006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,1023,0.03870933254559835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,1023,0.03849600007136663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,1023,0.038506666819254555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,1023,0.03793066740036011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,2047,0.04530133306980133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,2047,0.04684799909591675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,2047,0.04438399771849314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,2047,0.055871998270352684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,2047,0.046096002062161766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,2047,0.05153066913286845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,2047,0.04419200122356415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,2047,0.056048000852266945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,2047,0.05166399975617727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,2047,0.046122665206591286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,2047,0.045663997530937195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,2047,0.04484800000985464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,2047,0.044112001856168113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,4095,0.07214933137098949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,2047,0.04381333291530609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,4095,0.06464000046253204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,4095,0.06684266527493794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,4095,0.06389333307743073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,4095,0.0625546673933665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,4095,0.06296533346176147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,4095,0.06216000020503998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,4095,0.07206933200359344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,4095,0.06679999828338623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,4095,0.06459199885527293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,4095,0.06418133278687795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,4095,0.06247466802597046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,4095,0.062314664324124656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,4095,0.06272000074386597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,8191,0.09300800164540608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,8191,0.09850666920344035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,8191,0.08916266759236653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,8191,0.08853333195050557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,8191,0.0876533289750417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,8191,0.0869760016600291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,8191,0.08643733461697896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,8191,0.09946667154630025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,8191,0.09291733304659526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,8191,0.08969600001970927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,8191,0.08877333005269368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,8191,0.08690133690834045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,8191,0.08629866441090901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,8191,0.08648533622423808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,16383,0.1474186678727468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,16383,0.13609066605567932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,16383,0.13901866475741068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,16383,0.13503467043240866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,16383,0.13401599725087485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,16383,0.13359999656677246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,16383,0.1337493360042572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,16383,0.14754133423169455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,16383,0.13970667123794556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,16383,0.13417067130406699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,16383,0.1360106666882833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,16383,0.13357333342234293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,16383,0.13385599851608276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,16383,0.13379200299580893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,32767,0.24586133162180582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,32767,0.2350133260091146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,32767,0.2316640019416809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,32767,0.22999467452367148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,32767,0.22934399048487344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,32767,0.22816532850265503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,32767,0.22802132368087769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,32767,0.2454986572265625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,32767,0.2355146606763204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,32767,0.2321760058403015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,32767,0.22972800334294638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,32767,0.22959999243418375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,32767,0.22894400358200073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,32767,0.22850133975346884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,65535,0.4347039858500163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,65535,0.4230826695760091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,65535,0.41936532656351727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,65535,0.41661866505940753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,65535,0.41811732451121014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,65535,0.4160799980163574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,65535,0.4157866636912028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,65535,0.4357440074284871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,65535,0.42401599884033203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,65535,0.4202186663945516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,65535,0.41816532611846924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,65535,0.41653867562611896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,65535,0.41650664806365967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,65535,0.4174079895019531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,2,131071,0.7998080253601074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,1,131071,0.8145600159962972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,4,131071,0.7945333321889242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,8,131071,0.7926452954610189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,16,131071,0.7912586530049642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,32,131071,0.7906826337178549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,64,131071,0.7904480298360189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,1,131071,0.8147466977437338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,2,131071,0.8001546859741211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,8,131071,0.7919519742329916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,4,131071,0.7944959799448649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,32,131071,0.7918879985809326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,16,131071,0.7905653317769369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,64,131071,0.7912373542785645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,1,0.03317866722742716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,1,0.030495998760064442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,1,0.03154666721820831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,1,0.030938667555650074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,1,0.030784000953038532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,1,0.0303413321574529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,1,0.030394665896892548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,1,0.03323200096686681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,1,0.03142933299144109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,1,0.0308693324526151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,1,0.030453334252039593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,1,0.030597334106763203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,1,0.03029866764942805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,3,0.03311466674009959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,1,0.030432000756263733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,3,0.03145600110292435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,3,0.030943999687830608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,3,0.03031466652949651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,3,0.03033066789309184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,3,0.030645333230495453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,3,0.030378667016824085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,3,0.033402666449546814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,3,0.031445334355036415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,3,0.031040000418821972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,3,0.030394665896892548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,3,0.030437332888444264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,3,0.030394665896892548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,3,0.030421334008375805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,7,0.03331200033426285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,7,0.0314026673634847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,7,0.030608000854651134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,7,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,7,0.030165334542592365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,7,0.030218665798505146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,7,0.030005333324273426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,7,0.03312533348798752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,7,0.03352533280849457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,7,0.030602666238943737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,7,0.03019733230272929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,7,0.03029866764942805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,7,0.03009066730737686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,7,0.029951999584833782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,15,0.03088533381621043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,15,0.032842665910720825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,15,0.03018666555484136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,15,0.029701332251230877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,15,0.029893333713213604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,15,0.02976000060637792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,15,0.029722665747006733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,15,0.032799998919169106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,15,0.0313226655125618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,15,0.030154667794704437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,15,0.029781334102153778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,15,0.029850666721661884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,15,0.029898665845394135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,15,0.029690665503342945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,31,0.03262399882078171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,31,0.030847998956839245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,31,0.0296426663796107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,31,0.02924266705910365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,31,0.02942933390537898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,31,0.029445332785447437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,31,0.029178666571776073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,31,0.03239466746648153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,31,0.03081600119670232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,31,0.029669334491093952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,31,0.029253333806991577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,31,0.02923733244339625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,31,0.029391999046007793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,31,0.029157333076000214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,63,0.03229333211978277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,63,0.030218665798505146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,63,0.02886933336655299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,63,0.028362666567166645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,63,0.028229333460330963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,63,0.028368001182874043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,63,0.02827200045188268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,63,0.032255999743938446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,63,0.02826666583617528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,63,0.030266667405764263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,63,0.029050665597120922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,63,0.028351999819278717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,63,0.02842666705449422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,63,0.028336000939210255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,127,0.03469866762558619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,127,0.03173866619666418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,127,0.031109333038330078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,127,0.031328000128269196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,127,0.03233599911133448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,127,0.030991998811562855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,127,0.03107733279466629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,127,0.03501333296298981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,127,0.03276800115903219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,127,0.03140799949566523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,127,0.030933332939942677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,127,0.030981334547201794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,127,0.031045332551002502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,127,0.0308746670683225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,255,0.04176533222198486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,255,0.03682133307059606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,255,0.03665599972009659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,255,0.0382080003619194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,255,0.03668266783157984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,255,0.036864000062147774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,255,0.04165866722663244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,255,0.0364533339937528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,255,0.03782933453718821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,255,0.03699733316898346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,255,0.03704000016053518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,255,0.03668266783157984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,255,0.036789332826932274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,511,0.04497066636880239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,255,0.036650667587916054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,511,0.039994666973749794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,511,0.039103999733924866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,511,0.03877866764863332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,511,0.0390133336186409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,511,0.03818133225043615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,511,0.03984000037113825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,511,0.04515199859937032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,511,0.038202665746212006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,511,0.03939199944337209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,511,0.03907199949026108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,511,0.038245332737763725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,511,0.03829866647720337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,511,0.038176000118255615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,1023,0.05765866736570994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,1023,0.051642666260401406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,1023,0.047983999053637184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,1023,0.045791998505592346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,1023,0.046223998069763184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,1023,0.04527466495831808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,1023,0.045642669002215065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,1023,0.05825600028038025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,1023,0.05184000233809153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,1023,0.04791999856630961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,1023,0.046522667010625206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,1023,0.04527466495831808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,1023,0.045696000258127846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,1023,0.045466666420300804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,2047,0.07421866556008656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,2047,0.06762133538722992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,2047,0.06545066833496094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,2047,0.0643146683772405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,2047,0.06377066671848297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,2047,0.06295999884605408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,2047,0.06294933458169301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,2047,0.06803733110427856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,2047,0.06518933176994324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,2047,0.0738506664832433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,2047,0.0642080008983612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,2047,0.06358933448791504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,2047,0.06294399996598561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,2047,0.06277333199977875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,4095,0.09857599933942159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,4095,0.09222933650016785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,4095,0.08897067109743755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,4095,0.08798399567604065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,4095,0.08755733569463094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,4095,0.08624000350634257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,4095,0.08661866188049316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,4095,0.09894933303197224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,4095,0.089519997437795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,4095,0.09261332949002583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,4095,0.08820266524950664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,4095,0.08776533603668213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,4095,0.08672533432642619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,4095,0.08646399776140849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,8191,0.14749333262443542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,8191,0.14012266198794046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,8191,0.13615999619166055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,8191,0.13665599624315897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,8191,0.1344319979349772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,8191,0.13460800051689148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,8191,0.13422933220863342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,8191,0.1463093360265096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,8191,0.14006400108337402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,8191,0.1358026663462321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,8191,0.1371946632862091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,8191,0.1351093351840973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,8191,0.13482133547465006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,8191,0.13467199603716531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,16383,0.23939200242360434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,16383,0.23622934023539224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,16383,0.23057599862416586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,16383,0.2335253357887268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,16383,0.23047999540964761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,16383,0.22930665810902914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,16383,0.22894932826360068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,16383,0.24377065896987915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,16383,0.23207465807596842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,16383,0.23600532611211142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,16383,0.23045865694681802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,16383,0.229695995648702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,16383,0.22924266258875528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,16383,0.22946133216222128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,32767,0.42720532417297363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,32767,0.4423733154932658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,32767,0.42316265900929767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,32767,0.4194186528523763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,32767,0.41864534219106037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,32767,0.4187519947687785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,32767,0.41950400670369464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,32767,0.4400586684544881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,32767,0.43303465843200684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,32767,0.4231040080388387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,32767,0.41967999935150146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,32767,0.41927464803059894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,32767,0.4190560181935628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,32767,0.418506662050883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,65535,0.817413330078125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,65535,0.8116479714711508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,65535,0.7958026727040609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,65535,0.7922879854838053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,65535,0.793440024058024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,65535,0.7913653055826823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,65535,0.7912213007609049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,65535,0.8149226506551107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,65535,0.8130826950073242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,65535,0.7966506481170654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,65535,0.7934559981028239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,65535,0.7917439937591553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,65535,0.7912053267161051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,65535,0.7942026456197103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,1,131071,1.5622827212015789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,2,131071,1.56768004099528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,4,131071,1.5450879732767742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,8,131071,1.543824036916097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,16,131071,1.5402666727701824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,32,131071,1.5387040774027507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,64,131071,1.5402132670084636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,1,131071,1.5638826688130696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,8,131071,1.5421172777811687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,2,131071,1.567797342936198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,4,131071,1.5437493324279785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,16,131071,1.539482593536377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,1,0.039135999977588654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,32,131071,1.5381546020507812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,1,0.03611200054486593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,64,131071,1.5399413108825684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,1,0.03521066655715307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,1,0.03500800083080927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,1,0.03458133339881897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,1,0.0348693331082662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,1,0.03895466774702072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,1,0.03438399980465571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,1,0.03505066782236099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,1,0.03513066718975703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,1,0.036117332677046456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,1,0.034517332911491394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,1,0.03463999927043915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,1,0.034517332911491394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,3,0.03885866701602936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,3,0.03623466690381368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,3,0.035002666215101876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,3,0.03435199956099192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,3,0.03480533262093862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,3,0.03425066669782003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,3,0.03443199892838796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,3,0.038762666285037994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,3,0.03621333340803782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,3,0.034927998979886375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,3,0.03455466777086258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,3,0.03455466777086258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,3,0.034341332813103996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,3,0.03439466655254364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,7,0.03868266691764196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,7,0.036117332677046456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,7,0.034917332231998444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,7,0.03435733417669932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,7,0.03440533330043157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,7,0.034245334565639496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,7,0.03428266694148382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,7,0.03871466716130575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,7,0.03605333218971888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,7,0.03438399980465571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,7,0.035029334326585136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,7,0.03440533330043157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,7,0.03418133407831192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,7,0.0342399999499321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,15,0.03835733234882355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,15,0.03589333345492681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,15,0.034058667719364166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,15,0.03442666679620743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,15,0.03376533339420954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,15,0.033802665770053864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,15,0.033813332517941795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,15,0.03823466598987579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,15,0.035536001125971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,15,0.03429333368937174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,15,0.03403199960788091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,15,0.03378133227427801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,15,0.03389866650104523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,15,0.03389866650104523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,31,0.037647999823093414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,31,0.03366933266321818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,31,0.03514666606982549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,31,0.03381866713364919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,31,0.03334933271010717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,31,0.033173332611719765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,31,0.033376000821590424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,31,0.03533333291610082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,31,0.037871999045213066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,31,0.03392533212900162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,31,0.03335466732581457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,31,0.03346133232116699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,31,0.033359999457995095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,31,0.03332266708215078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,63,0.037674665451049805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,63,0.03303466737270355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,63,0.03452266752719879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,63,0.03271466741959254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,63,0.032357332607110344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,63,0.032261334359645844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,63,0.032325332363446556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,63,0.03730133424202601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,63,0.03460799902677536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,63,0.03306666761636734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,63,0.03264000018437704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,63,0.032314665615558624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,63,0.03232000023126602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,127,0.03997866561015447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,63,0.032314665615558624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,127,0.03718400001525879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,127,0.03562666724125544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,127,0.035760000348091125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,127,0.03545066714286804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,127,0.03510399907827377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,127,0.03545066714286804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,127,0.04063999901215235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,127,0.0373333344856898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,127,0.035829332967599235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,127,0.03568533311287562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,127,0.03551466763019562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,127,0.03522666543722153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,127,0.03530666728814443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,255,0.039408000806967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,255,0.04121600091457367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,255,0.04400533437728882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,255,0.03931200007597605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,255,0.03967999915281931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,255,0.03915199885765711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,255,0.039093332986036934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,255,0.04385066529115041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,255,0.040181333820025124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,255,0.040789333482583366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,255,0.03935466706752777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,255,0.039488000174363456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,255,0.03905600061019262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,255,0.03922666609287262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,511,0.06570666531721751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,511,0.05758399764696757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,511,0.05292266607284546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,511,0.04994133114814758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,511,0.04919999837875366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,511,0.05796800057093302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,511,0.04949333270390829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,511,0.04921066761016846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,511,0.05057600140571594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,511,0.06537599861621857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,511,0.052986666560173035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,511,0.04948799808820089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,511,0.0492799977461497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,511,0.049600000182787575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,1023,0.08562133709589641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,1023,0.06917333106199901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,1023,0.06677866478761037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,1023,0.07461866736412048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,1023,0.06801066795984904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,1023,0.06502399841944377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,1023,0.06644266843795776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,1023,0.08524266878763835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,1023,0.0745119998852412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,1023,0.06937066713968913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,1023,0.06726400057474773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,1023,0.06730666756629944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,1023,0.06582933167616527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,1023,0.06547733147939046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,2047,0.10734933614730835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,2047,0.0979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,2047,0.09277333815892537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,2047,0.09145067135492961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,2047,0.09002133210500081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,2047,0.09031466643015544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,2047,0.0902453362941742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,2047,0.10681066910425822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,2047,0.09776533643404643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,2047,0.09365333120028178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,2047,0.0913759966691335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,2047,0.09018666545550029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,2047,0.0899679958820343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,2047,0.09082133571306865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,4095,0.1460586686929067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,4095,0.15503467122713724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,4095,0.13923199971516928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,4095,0.14011200269063315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,4095,0.137855996688207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,4095,0.13781332969665527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,4095,0.13802666465441385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,4095,0.1541973352432251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,4095,0.14538666605949402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,4095,0.1405173341433207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,4095,0.13886933525403342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,4095,0.1386613349119822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,4095,0.1381386617819468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,4095,0.13854933778444925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,8191,0.2507946689923604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,8191,0.24093333880106607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,8191,0.23313599824905396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,8191,0.2320693333943685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,8191,0.2346400022506714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,8191,0.23439999421437582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,8191,0.2315679987271627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,8191,0.24077866474787393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,8191,0.25110934178034466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,8191,0.23464532693227133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,8191,0.23335466782251993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,8191,0.23254932959874472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,8191,0.23489065965016684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,8191,0.23194666703542074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,16383,0.4536426862080892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,16383,0.4383840163548787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,16383,0.42797334988911945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,16383,0.42421332995096844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,16383,0.42133867740631104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,16383,0.4214239915211995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,16383,0.42372798919677734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,16383,0.455946683883667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,16383,0.43881066640218097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,16383,0.42796798547108966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,16383,0.42124799887339276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,16383,0.42390934626261395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,16383,0.4291733503341675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,16383,0.4212053219477336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,32767,0.8325280348459879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,32767,0.8141226768493652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,32767,0.8022080262502035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,32767,0.7999626795450846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,32767,0.7960000038146973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,32767,0.7981332937876383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,32767,0.8028852939605713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,32767,0.8301653067270914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,32767,0.8155946731567383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,32767,0.797871987024943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,32767,0.8042186896006266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,32767,0.796837329864502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,32767,0.795039971669515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,32767,0.8066133658091227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,65535,1.5808107058207195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,65535,1.5600800514221191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,65535,1.5503679911295574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,65535,1.5448692639668782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,65535,1.5420799255371094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,65535,1.5480480194091797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,65535,1.5605120658874512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,65535,1.5826773643493652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,65535,1.5604267120361328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,65535,1.5524427096048992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,65535,1.547141393025716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,65535,1.5409493446350098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,65535,1.5460426012674968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,65535,1.561914602915446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,1,131071,3.075968106587728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,4,131071,3.0426772435506186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,16,131071,3.036229451497396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,8,131071,3.0389226277669272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,2,131071,3.0543734232584634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,64,131071,3.078181266784668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,32,131071,3.0369545618693032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,1,131071,3.0763254165649414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,2,131071,3.055178642272949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,1,0.04886400202910105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,32,131071,3.074703852335612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,8,131071,3.039456049601237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,4,131071,3.0423145294189453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,16,131071,3.0378986994425454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,64,131071,3.0372533798217773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,1,0.044938668608665466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,1,0.042954668402671814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,1,0.0417546679576238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,1,0.04155733436346054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,1,0.04156800111134847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,1,0.04889066517353058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,1,0.04301866888999939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,1,0.0415040006240209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,1,0.044639999667803444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,1,0.041519999504089355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,1,0.041690667470296226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,1,0.04153066625197729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,1,0.04155733436346054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,3,0.04261866708596548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,3,0.04478399952252706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,3,0.04862933357556661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,3,0.04189866781234741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,3,0.04128533353408178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,3,0.04151466737190882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,3,0.04156800111134847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,3,0.0487306664387385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,3,0.044906665881474815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,3,0.042650664846102394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,3,0.04189866781234741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,3,0.04126933217048645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,3,0.04145599901676178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,3,0.04155199974775314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,7,0.048613334695498146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,7,0.04264000058174133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,7,0.04453866680463155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,7,0.04154666761557261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,7,0.04114133367935816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,7,0.04133866727352142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,7,0.04121600091457367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,7,0.04462933540344238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,7,0.042624001701672874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,7,0.04857600231965383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,7,0.04156800111134847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,7,0.041034666200478874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,7,0.041237334410349526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,7,0.041306667029857635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,15,0.048021331429481506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,15,0.044256001710891724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,15,0.04204800228277842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,15,0.04125866790612539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,15,0.04074666649103165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,15,0.04083200047413508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,15,0.04091199984153112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,15,0.0480320006608963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,15,0.044256001710891724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,15,0.04124266654253006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,15,0.0421973317861557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,15,0.04095999896526337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,15,0.04083200047413508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,15,0.040896000961462654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,31,0.04780266682306925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,31,0.04379733403523763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,31,0.04060266663630804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,31,0.04172799984614054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,31,0.040922666589419045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,31,0.040362666050593056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,31,0.04033066580692927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,31,0.04796266555786133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,31,0.04378666480382284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,31,0.041722665230433144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,31,0.04073066761096319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,31,0.04065600037574768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,31,0.04008533308903376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,63,0.047882666190465294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,63,0.04322133461634318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,31,0.04045333216587702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,63,0.041093334555625916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,63,0.039994666973749794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,63,0.039221333960692085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,63,0.039274667700131737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,63,0.03953066716591517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,63,0.047882666190465294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,63,0.04322666426499685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,63,0.04115733255942663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,63,0.03990400085846583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,63,0.0395413339138031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,63,0.03924266745646795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,127,0.051856001218159996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,63,0.03934400031963984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,127,0.04379733403523763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,127,0.04295999805132548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,127,0.0461760014295578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,127,0.04258666435877482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,127,0.04242133100827535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,127,0.04262933135032654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,127,0.052501335740089417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,127,0.04620266457398733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,127,0.04401599864164988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,127,0.04317333300908407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,127,0.04251733422279358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,127,0.042463997999827065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,127,0.04269866645336151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,255,0.05670933425426483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,255,0.06332799792289734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,255,0.06294399996598561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,255,0.047024001677831016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,255,0.0480320006608963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,255,0.04711466530958811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,255,0.04773333172003428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,255,0.06380799909432729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,255,0.05204799771308899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,255,0.056741332014401756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,255,0.04747200012207031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,255,0.047210668524106346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,255,0.04773333172003428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,255,0.047983999053637184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,511,0.0769706666469574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,511,0.07088000078996022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,511,0.06797866523265839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,511,0.06674666702747345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,511,0.06592533489068349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,511,0.06560533245404561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,511,0.06564266482988994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,511,0.07692266503969829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,511,0.0707946668068568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,511,0.06825600067774455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,511,0.06625066697597504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,511,0.06580266853173573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,511,0.06665599842866261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,511,0.06540266672770183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,1023,0.10079999764760335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,1023,0.09499733646710713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,1023,0.0902400016784668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,1023,0.09130133191744487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,1023,0.0897226631641388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,1023,0.0890773336092631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,1023,0.08885332942008972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,1023,0.10151466727256775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,1023,0.09161600470542908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,1023,0.09482666850090027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,1023,0.09012800455093384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,1023,0.08996267120043437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,1023,0.08930133779843648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,1023,0.0895253320535024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,2047,0.1431946655114492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,2047,0.14969066778818765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,2047,0.13843733072280884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,2047,0.13963733116785684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,2047,0.13801067074139914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,2047,0.1374613344669342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,2047,0.1372213363647461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,2047,0.14288000265757242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,2047,0.14990933736165366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,2047,0.13844799995422363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,2047,0.1381173332532247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,2047,0.13939199844996134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,2047,0.13710932930310568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,2047,0.13693333665529886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,4095,0.24521066745122275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,4095,0.23839465777079263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,4095,0.2359679937362671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,4095,0.23238933086395264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,4095,0.23490667343139648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,4095,0.2322346568107605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,4095,0.2321866750717163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,4095,0.23764799038569132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,4095,0.24494399627049765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,4095,0.23545066515604654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,4095,0.23373333613077799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,4095,0.23312000433603922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,4095,0.23350399732589722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,4095,0.23162132501602173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,8191,0.42847998936971027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,8191,0.43516798814137775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,8191,0.4222186803817749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,8191,0.42370132605234784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,8191,0.4214506546656291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,8191,0.4212053219477336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,8191,0.42191465695699054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,8191,0.4344319899876912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,8191,0.42765335241953534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,8191,0.42558932304382324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,8191,0.4230026801427205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,8191,0.42102932929992676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,8191,0.4215199947357178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,8191,0.4227360089619954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,16383,0.858954668045044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,16383,0.8311253388722738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,16383,0.8078186511993408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,16383,0.8210773468017578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,16383,0.8042773405710856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,16383,0.8004426956176758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,16383,0.8023626804351807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,16383,0.8565119902292887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,16383,0.8294560114542643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,16383,0.8127520084381104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,16383,0.8163572947184244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,16383,0.8033599853515625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,16383,0.8000533580780029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,16383,0.8033920129140218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,32767,1.6095892588297527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,32767,1.5804266929626465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,32767,1.5784586270650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,32767,1.5516640345255535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,32767,1.545973300933838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,32767,1.5481492678324382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,32767,1.555888017018636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,32767,1.6105119387308757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,32767,1.579978624979655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,32767,1.5529279708862305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,32767,1.5786399841308594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,32767,1.547594706217448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,32767,1.5475093523661296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,32767,1.5530452728271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,65535,3.106186548868815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,65535,3.076047897338867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,65535,3.056495984395345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,65535,3.0838454564412436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,65535,3.0393174489339194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,65535,3.0405654907226562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,65535,3.041269302368164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,65535,3.10810120900472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,65535,3.074517250061035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,65535,3.044618606567383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,65535,3.0907627741495767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,65535,3.03985595703125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,65535,3.037893295288086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,65535,3.044975916544596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,1,131071,6.09552001953125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,2,131071,6.058522542317708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,16,131071,6.023242950439453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,8,131071,6.10645866394043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,64,131071,6.027477264404297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,4,131071,6.0422719319661455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,32,131071,6.02230962117513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,1,131071,6.093674977620442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,2,131071,6.058821360270183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,1,0.07050666709740956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,1,0.0610346645116806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,1,0.05771199862162272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,1,0.05602133274078369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,1,0.055434669057528176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,1,0.05482666691144308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,1,0.05486933390299479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,32,131071,6.024410883585612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,4,131071,6.039674758911133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,8,131071,6.104170481363933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,1,0.07032533486684163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,1,0.06092800199985504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,64,131071,6.028229395548503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,1,0.0576853354771932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,1,0.05527466535568237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,1,0.054773335655530296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,1,0.05606399973233541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,1,0.05477866530418396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,16,131071,6.022975921630859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,3,0.057392001152038574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,3,0.05583466589450836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,3,0.07022933165232341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,3,0.055120001236597695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,3,0.060880000392595925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,3,0.05467733244101206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,3,0.05464000006516775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,3,0.057333335280418396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,3,0.06090133388837179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,3,0.07043200234572093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,3,0.055674667159716286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,3,0.05436266462008158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,3,0.05504000186920166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,3,0.05470933516820272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,7,0.07045333087444305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,7,0.05692266424496969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,7,0.06062933305899302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,7,0.05537599821885427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,7,0.05426133175690969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,7,0.054330666859944664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,7,0.05426666637261709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,7,0.060565332571665444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,7,0.05712000032265981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,7,0.07043733199437459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,7,0.05540800094604492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,7,0.05433600147565206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,7,0.05433600147565206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,7,0.054416000843048096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,15,0.07138133545716603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,15,0.056032001972198486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,15,0.05973866581916809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,15,0.054474666714668274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,15,0.053488001227378845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,15,0.05319466690222422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,15,0.05339199801286062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,15,0.07170133292675018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,15,0.05622933308283488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,15,0.05975999931494395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,15,0.0545066644748052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,15,0.05341866612434387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,15,0.053264002005259194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,15,0.053360000252723694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,31,0.0738560010989507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,31,0.059205333391825356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,31,0.05509866774082184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,31,0.05239999790986379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,31,0.05229333539803823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,31,0.05349333087603251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,31,0.052527998884518944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,31,0.07372266550858815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,31,0.059088001648585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,31,0.0535093347231547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,31,0.05267733335494995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,31,0.05509866774082184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,31,0.05248000224431356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,31,0.052469333012898765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,63,0.06019733349482218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,63,0.05388799806435903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,63,0.07761600116888683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,63,0.05100266635417938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,63,0.05091733237107595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,63,0.051776001850763954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,63,0.05132266879081726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,63,0.06026133398214976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,63,0.07743999858697255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,63,0.05137600004673004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,63,0.05223466455936432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,63,0.050698667764663696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,63,0.0539680023988088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,63,0.05055999755859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,127,0.06578133503595988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,127,0.08283733328183492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,127,0.07188799977302551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,127,0.060959999759991966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,127,0.05729599793752035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,127,0.056890666484832764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,127,0.058117335041364036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,127,0.07181866466999054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,127,0.08273066580295563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,127,0.0650079995393753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,127,0.0609386662642161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,127,0.057376002271970115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,127,0.057301332553227745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,127,0.05789333085219065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,255,0.08486933509508769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,255,0.0788373351097107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,255,0.07653333246707916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,255,0.07509866853555043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,255,0.09649067123730977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,255,0.07408533493677776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,255,0.07437866429487865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,255,0.09666666388511658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,255,0.07901333272457123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,255,0.08443733056386311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,255,0.07658133407433827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,255,0.07445866862932841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,255,0.07380266487598419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,255,0.07552533348401387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,511,0.11030399799346924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,511,0.10384533802668254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,511,0.12350933750470479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,511,0.10022399822870891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,511,0.0985599954922994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,511,0.10020266969998677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,511,0.12302933136622111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,511,0.10108266274134318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,511,0.10949333508809407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,511,0.10410666465759277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,511,0.09778666496276855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,511,0.0990666647752126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,511,0.09923199812571208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,511,0.09865599870681763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,1023,0.17107200622558594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,1023,0.1569493313630422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,1023,0.14604266484578451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,1023,0.1513599952061971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,1023,0.14526933431625366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,1023,0.1481653352578481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,1023,0.14426133036613464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,1023,0.15061333775520325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,1023,0.14596266547838846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,1023,0.1580586632092794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,1023,0.17111466328303018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,1023,0.14789866407712302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,1023,0.14477866888046265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,1023,0.145797332127889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,2047,0.24178133408228555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,2047,0.25270400444666546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,2047,0.26503467559814453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,2047,0.24417599042256674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,2047,0.24100265900293985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,2047,0.23917333285013834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,2047,0.26578666766484577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,2047,0.2414240042368571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,2047,0.23926933606465658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,2047,0.24507200717926025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,2047,0.2400266726811727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,2047,0.25145600239435834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,2047,0.23887999852498373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,2047,0.23941866556803384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,4095,0.42821868260701496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,4095,0.4542773167292277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,4095,0.4356906811396281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,4095,0.4300853411356608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,4095,0.43070932229359943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,4095,0.44175998369852704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,4095,0.4287946621576945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,4095,0.4538293282190959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,4095,0.435424009958903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,4095,0.43884801864624023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,4095,0.43188265959421795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,4095,0.4300533135732015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,4095,0.4285920063654582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,4095,0.4296106497446696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,8191,0.8749333222707113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,8191,0.8335466384887695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,8191,0.814197301864624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,8191,0.809279998143514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,8191,0.8431893189748129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,8191,0.8107093175252279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,8191,0.8128960132598877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,8191,0.8772853215535482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,8191,0.8284853299458822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,8191,0.8498506546020508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,8191,0.8106559912363688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,8191,0.8147146701812744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,8191,0.8091999689737955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,8191,0.8139946460723877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,16383,1.6342132886250813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,16383,1.598629315694173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,16383,1.5939359664916992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,16383,1.565285364786784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,16383,1.559765338897705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,16383,1.5552746454874675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,16383,1.5614293416341145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,16383,1.6334880193074544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,16383,1.5988160769144695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,16383,1.5948479970296223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,16383,1.5644373893737793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,16383,1.5572266578674316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,16383,1.5548639297485352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,16383,1.5627733866373699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,32767,3.1372801462809243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,32767,3.0963786443074546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,32767,3.10810120900472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,32767,3.050213177998861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,32767,3.0488319396972656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,32767,3.0581385294596353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,32767,3.056565284729004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,32767,3.1380106608072915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,32767,3.1093918482462564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,32767,3.0567092895507812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,32767,3.10097599029541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,32767,3.0515359242757163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,32767,3.047445297241211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,32767,3.054501215616862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,2,65535,6.08469835917155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,8,65535,6.040138880411784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,1,65535,6.19486935933431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,16,65535,6.033407847086589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,4,65535,6.054874420166016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,32,65535,6.030122756958008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,64,65535,6.040810902913411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,1,65535,6.126581192016602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,2,65535,6.089471817016602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,4,65535,6.125136057535808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,8,65535,6.038997650146484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,16,65535,6.034218470255534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,32,65535,6.041882832845052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,1,0.11523200074831645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,1,0.0869653324286143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,1,0.09595732887585957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,1,0.08359466989835103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,1,0.08092799782752991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,1,0.08191466828187306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,1,0.08130666613578796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,1,0.11547733346621196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,1,0.08693333466847737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,1,0.08187733093897502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,1,0.08320533235867818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,1,0.081386665503184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,1,0.09571733077367146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,1,0.0812853326400121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,3,0.1141813298066457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,3,0.08661866188049316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,3,0.08305599788824718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,3,0.09551466504732768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,3,0.08119999865690868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,3,0.0819413314263026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,3,0.08100266754627228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,3,0.08316799998283386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,3,0.08666132887204488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,3,0.09567999839782715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,3,0.11390933394432068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,3,0.08132266501585643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,3,0.08198399841785431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,3,0.08088000118732452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,7,0.08583999673525493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,7,0.11405866344769795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,7,0.09524800380071004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,7,0.0824480007092158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,7,0.08063466846942902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,64,65535,6.0436585744222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,7,0.08021333316961925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,7,0.0812853326400121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,7,0.09526399771372478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,7,0.11402133107185364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,7,0.08246933420499165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,7,0.0858026643594106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,7,0.08094933132330577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,7,0.08021866778532664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,7,0.08066133161385854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,15,0.11402666568756104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,15,0.08429333567619324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,15,0.08072533210118611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,15,0.09458667039871216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,15,0.07920533418655396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,15,0.0786293347676595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,15,0.07829866806666057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,15,0.08398399750391643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,15,0.09449600179990132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,15,0.11416533589363098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,15,0.07925866544246674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,15,0.08096533517042796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,15,0.07860266665617625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,15,0.07834133505821228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,31,0.09622933467229207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,31,0.11626666784286499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,31,0.07860266665617625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,31,0.08344533046086629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,31,0.0774239997069041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,31,0.0767146646976471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,31,0.0763679991165797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,31,0.09566932916641235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,31,0.07858133316040039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,31,0.07740800082683563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,31,0.08344533046086629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,31,0.11619733770688374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,31,0.07663466533025105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,31,0.07653333246707916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,63,0.08841600020726521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,63,0.12090133627255757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,63,0.08020266890525818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,63,0.07576533158620198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,63,0.08340799808502197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,63,0.09935466448465984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,63,0.07487999896208446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,63,0.09949866930643718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,63,0.1206719974676768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,63,0.0835040012995402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,63,0.0792746643225352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,63,0.08853333195050557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,63,0.07575466732184093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,63,0.07373866438865662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,127,0.13272533814112344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,127,0.11166933178901672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,127,0.0988106628259023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,127,0.09361599882443745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,127,0.09047999978065491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,127,0.0918986697991689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,127,0.09092799822489421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,127,0.11096533139546712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,127,0.13221333424250284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,127,0.09930666287740071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,127,0.09413333733876546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,127,0.09243733684221904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,127,0.09050666292508443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,127,0.0902346670627594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,255,0.13592533270517984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,255,0.11597333351771037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,255,0.15743466218312582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,255,0.1244053343931834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,255,0.11885866522789001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,255,0.11532266934712727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,255,0.11398933331171672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,255,0.13454932967821756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,255,0.12446399529774983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,255,0.1185653309027354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,255,0.15738667050997415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,255,0.11565867066383362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,255,0.11456533273061116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,255,0.1139359970887502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,511,0.17312532663345337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,511,0.16661866505940756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,511,0.16170666615168253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,511,0.2090346614519755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,511,0.18542933464050293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,511,0.1637333333492279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,511,0.16089066863059998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,511,0.20904000600179037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,511,0.16447466611862183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,511,0.1845653255780538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,511,0.1604266663392385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,511,0.17333867152531943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,511,0.1609546641508738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,511,0.16666133205095926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,1023,0.2566986680030823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,1023,0.30586665868759155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,1023,0.2798080046971639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,1023,0.2563520073890686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,1023,0.2605546712875366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,1023,0.2672533392906189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,1023,0.2540480097134908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,1023,0.3060373266537984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,1023,0.25524266560872394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,1023,0.2571306626001994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,1023,0.267792006333669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,1023,0.2612160046895345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,1023,0.27962666749954224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,1023,0.25376532475153607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,2047,0.4466293255488078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,2047,0.46935999393463135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,2047,0.4939200083414714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,2047,0.4500693480173747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,2047,0.44354132811228436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,2047,0.4561226765314738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,2047,0.4418506622314453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,2047,0.45707201957702637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,2047,0.4683413505554199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,2047,0.4459253152211507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,2047,0.4495893319447835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,2047,0.4938400189081828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,2047,0.4445759852727254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,2047,0.44202665487925213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,4095,0.8743893305460612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,4095,0.9131999810536703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,4095,0.8363626797993978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,4095,0.8500800132751465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,4095,0.835424025853475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,4095,0.8257493178049723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,4095,0.8222933610280355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,4095,0.8740213712056478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,4095,0.8377760251363119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,4095,0.8347946802775065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,4095,0.8232106367746989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,4095,0.9125333627065023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,4095,0.8523093064626058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,4095,0.8226133187611898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,8191,1.5736586252848308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,8191,1.594879945119222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,8191,1.6870773633321126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,8191,1.617509365081787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,8191,1.6344693501790364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,8191,1.568069299062093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,8191,1.5703360239664714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,8191,1.604383945465088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,8191,1.6029653549194336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,8191,1.5746453603108723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,8191,1.6815306345621746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,8191,1.6354400316874187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,8191,1.5687732696533203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,8191,1.5723412831624348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,16383,3.1117493311564126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,16383,3.081242561340332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,16383,3.188277244567871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,16383,3.1675148010253906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,16383,3.0667947133382163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,16383,3.0650558471679688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,16383,3.06874148050944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,16383,3.1324691772460938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,16383,3.103359858194987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,16383,3.0676905314127603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,16383,3.114192008972168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,16383,3.1853065490722656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,16383,3.065312067667643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,16383,3.067157427469889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,1,32767,6.1910826365153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,2,32767,6.124346415201823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,4,32767,6.100517272949219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,8,32767,6.132432301839192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,32,32767,6.049173355102539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,16,32767,6.050191879272461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,64,32767,6.053845087687175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,2,32767,6.125674565633138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,1,32767,6.181568145751953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,4,32767,6.101381301879883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,8,32767,6.138832092285156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,1,0.16330666343371072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,1,0.19523733854293823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,16,32767,6.055370966593425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,1,0.14852799971898398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,1,0.1365066667397817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,1,0.13884799679120383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,1,0.13529599706331888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,1,0.13380799690882364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,32,32767,6.049770355224609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,1,0.13909866412480673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,1,0.16331733266512552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,1,0.14874133467674255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,1,0.1362666686375936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,1,0.1953386664390564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,1,0.13515200217564902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,1,0.1339040001233419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,3,0.13716800014177957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,3,0.161189337571462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,3,0.19348265727361044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,3,0.13457066814104715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,3,0.146997332572937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,3,0.13292800386746725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,3,0.13260799646377563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,3,0.14662933349609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,3,0.13737600048383078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,3,0.16125866770744324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,3,0.1345919966697693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,3,0.19335466623306274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,3,0.13337066769599915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,3,0.13269333044687906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,64,32767,6.055434544881185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,7,0.14475733041763306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,7,0.15993066628774008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,7,0.1926506757736206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,7,0.13617600003878275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,7,0.13312000036239624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,7,0.13180266817410788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,7,0.13063466548919678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,7,0.13572266697883606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,7,0.19314134120941162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,7,0.13184533516565958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,7,0.1451359987258911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,7,0.13267200191815695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,7,0.15980266531308493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,7,0.13057600458463034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,15,0.1349493364493052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,15,0.14296000202496847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,15,0.12874133388201395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,15,0.13026133179664612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,15,0.15824533502260843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,15,0.19221866130828857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,15,0.12803199887275696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,15,0.1337386667728424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,15,0.1302773356437683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,15,0.1430239975452423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,15,0.12889599800109863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,15,0.12809066971143088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,15,0.15826132893562317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,15,0.19242133696873984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,31,0.13967466354370117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,31,0.12652800480524698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,31,0.156741331020991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,31,0.12854933738708496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,31,0.1309599975744883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,31,0.1946186621983846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,31,0.12550399700800577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,31,0.12813867131868997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,31,0.13168000181516012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,31,0.12602133552233377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,31,0.15666133165359497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,31,0.13994133472442627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,31,0.1251626710096995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,31,0.19602133830388388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,63,0.13327466448148093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,63,0.14178666472434998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,63,0.16079466541608176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,63,0.1270080010096232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,63,0.12970667084058127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,63,0.1271573305130005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,63,0.2074026664098104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,63,0.1418186624844869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,63,0.1620373328526815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,63,0.12748799721399942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,63,0.13148799538612366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,63,0.12601600090662637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,63,0.1297813355922699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,63,0.20653333266576132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,127,0.1472640037536621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,127,0.14496533075968424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,127,0.15288000305493674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,127,0.14406933387120566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,127,0.18669867515563965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,127,0.16395733753840128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,127,0.2311413288116455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,127,0.1516480048497518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,127,0.145306666692098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,127,0.16368533174196878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,127,0.18585066000620523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,127,0.14797332882881165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,127,0.1437173287073771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,127,0.2302773396174113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,255,0.20203733444213867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,255,0.19612266620000204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,255,0.21258666117986044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,255,0.19092265764872232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,255,0.2336746652921041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,255,0.1894986629486084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,255,0.27822933594385785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,255,0.21227200826009116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,255,0.20145066579182944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,255,0.19645865758260092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,255,0.19132800896962485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,255,0.27818665901819867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,255,0.23250667254130045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,255,0.18980799118677774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,511,0.3330559929211934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,511,0.28957333167394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,511,0.28593067328135174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,511,0.2972853382428487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,511,0.3104479908943176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,511,0.37885868549346924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,511,0.283077339331309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,511,0.2972373366355896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,511,0.3099466760953267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,511,0.3327573339144389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,511,0.28335465987523395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,511,0.28615466753641766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,511,0.37887465953826904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,511,0.2909653385480245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,1023,0.5001706679662069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,1023,0.4853173494338989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,1023,0.5240106582641602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,1023,0.4743146498998006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,1023,0.47834134101867676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,1023,0.5729920069376627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,1023,0.4723680019378662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,1023,0.5001333157221476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,1023,0.4870293140411377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,1023,0.47809068361918133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,1023,0.5242133140563965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,1023,0.5731360117594401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,1023,0.4730079968770345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,1023,0.4713546832402547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,2047,0.8731520175933838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,2047,0.8938506444295248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,2047,0.8611466884613037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,2047,0.9349706967671713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,2047,0.855685313542684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,2047,0.9861813386281332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,2047,0.8525546391805013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,2047,0.8765493233998617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,2047,0.8920799891153971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,2047,0.8608746528625488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,2047,0.9275893370310465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,2047,0.8539786338806152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,2047,0.983680009841919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,2047,0.85426131884257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,4095,1.7001439730326335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,4095,1.6543413798014324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,4095,1.6309760411580403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,4095,1.779274622599284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,4095,1.6001653671264648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,4095,1.6116320292154949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,4095,1.604650656382243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,4095,1.6548479398091633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,4095,1.6396106084187825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,4095,1.6983307202657063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,4095,1.7709867159525554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,4095,1.6109066009521484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,4095,1.6023999849955242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,4095,1.6019093195597331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,8191,3.1304906209309897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,8191,3.152016003926595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,8191,3.2350454330444336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,8191,3.296522776285807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,8191,3.1059999465942383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,8191,3.0959574381510415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,8191,3.095930735270182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,8191,3.2069387435913086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,8191,3.1784000396728516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,8191,3.294095993041992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,8191,3.1289440790812173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,8191,3.1102507909139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,8191,3.095888137817383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,8191,3.0991573333740234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,8,16383,6.111061096191406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,4,16383,6.145349502563477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,2,16383,6.269701639811198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,1,16383,6.307770411173503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,16,16383,6.09446398417155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,32,16383,6.079925537109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,64,16383,6.082880020141602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,1,16383,6.308277130126953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,8,16383,6.107952117919922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,2,16383,6.198917388916016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,4,16383,6.205738703409831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,16,16383,6.096687952677409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,1,0.022426667312781017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,1,0.027744000156720478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,1,0.026911998788515728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,1,0.0262719988822937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,32,16383,6.081178665161133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,1,0.026320000489552815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,1,0.0260959987839063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,1,0.026789332429567974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,1,0.022522665560245514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,1,0.027914665639400482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,1,0.02697066714366277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,1,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,1,0.026447998980681103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,1,0.026047999660174053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,1,0.026314665873845417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,3,0.022463999688625336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,3,0.027765333652496338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,3,0.026954665780067444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,3,0.026314665873845417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,3,0.025888000925381977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,3,0.026000000536441803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,3,0.026314665873845417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,3,0.02232533444960912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,3,0.027653334041436512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,3,0.026602665583292644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,3,0.026367999613285065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,3,0.026074667771657307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,3,0.02630399912595749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,7,0.026586666703224182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,7,0.022255999346574146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,7,0.02717866748571396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,3,0.026911998788515728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,7,0.026309333741664886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,7,0.025861332813898723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,7,0.025600001215934753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,7,0.02619733413060506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,7,0.02202133337656657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,7,0.027466667195161183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,7,0.026799999177455902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,7,0.026261332134405773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,7,0.02569066733121872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,7,0.025957333544890087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,7,0.02593066543340683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,64,16383,6.082794825236003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,15,0.021776000658671062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,15,0.027077332139015198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,15,0.026335999369621277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,15,0.02588266630967458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,15,0.025909334421157837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,15,0.02571200082699458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,15,0.02565866708755493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,15,0.021754667162895203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,15,0.027114666998386383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,15,0.026341333985328674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,31,0.021359999974568684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,31,0.026682667434215546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,15,0.025744001070658367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,15,0.02587199956178665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,15,0.025477332373460133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,15,0.026250667870044708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,31,0.02593066543340683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,31,0.025839999318122864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,31,0.025333332518736523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,31,0.025285333395004272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,31,0.025248001019159954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,31,0.021061333517233532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,31,0.026821332673231762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,31,0.025861332813898723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,31,0.025360000630219776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,31,0.025258667767047882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,31,0.025759999950726826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,31,0.025258667767047882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,63,0.02589866767326991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,63,0.020202666521072388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,63,0.02481599897146225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,63,0.024842667082945507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,63,0.02430933217207591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,63,0.024101334313551586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,63,0.024314666787783306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,63,0.020506666352351505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,63,0.025487999121348064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,63,0.024826665719350178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,63,0.024373332659403484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,63,0.02478933334350586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,63,0.024122667809327442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,127,0.023418667415777843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,127,0.027317332724730175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,63,0.02657066782315572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,127,0.02845866729815801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,127,0.02622399975856145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,127,0.026752000053723652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,127,0.026127999027570088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,127,0.026234666506449383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,127,0.023333333432674408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,127,0.028543998797734577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,127,0.02630399912595749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,127,0.027258666853109997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,127,0.026176000634829204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,127,0.026752000053723652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,255,0.02346133440732956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,127,0.02624000112215678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,255,0.027248000105222065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,255,0.02845866729815801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,255,0.026208000878492992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,255,0.0268053337931633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,255,0.02603733291228612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,255,0.026485333840052288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,255,0.023605334262053173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,255,0.028629332780838013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,255,0.027141332626342773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,255,0.026682667434215546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,255,0.026650667190551758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,255,0.026074667771657307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,511,0.024298667907714844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,511,0.029317334294319153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,255,0.026447998980681103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,511,0.027823999524116516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,511,0.02737066646416982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,511,0.027215999861558277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,511,0.029509333272775013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,511,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,511,0.026943999032179516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,511,0.02784000088771184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,511,0.027450665831565857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,511,0.024293333292007446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,511,0.026885333160559338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,511,0.026730666557947796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,511,0.026538667579491932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,1023,0.026202666262785595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,1023,0.030229332546393078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,1023,0.029002666473388672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,1023,0.028421332438786823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,1023,0.027855999767780304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,1023,0.027829334139823914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,1023,0.02770666778087616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,1023,0.026154667139053345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,1023,0.030261332790056866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,1023,0.028789333999156952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,1023,0.028565332293510437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,1023,0.02788266787926356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,1023,0.027674667537212372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,1023,0.027914665639400482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,2047,0.028463999430338543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,2047,0.033546666304270424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,2047,0.030746666093667347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,2047,0.03162133445342382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,2047,0.030117332935333252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,2047,0.03019733230272929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,2047,0.03052799900372823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,2047,0.033471999069054924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,2047,0.03182400017976761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,2047,0.030821333328882854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,2047,0.02845866729815801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,2047,0.030069333811601002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,2047,0.030229332546393078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,2047,0.030234667162100475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,4095,0.030565333863099415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,4095,0.03804266701141993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,4095,0.03608000030120214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,4095,0.035386666655540466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,4095,0.034245334565639496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,4095,0.03442133218050003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,4095,0.03428266694148382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,4095,0.03806933263937632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,4095,0.030586667358875275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,4095,0.03601066768169403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,4095,0.03543466577927271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,4095,0.03428266694148382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,4095,0.03442666679620743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,4095,0.034272000193595886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,8191,0.03428266694148382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,8191,0.040181333820025124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,8191,0.03782933453718821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,8191,0.03716800113519033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,8191,0.03643200049797694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,8191,0.03429333368937174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,8191,0.0366239994764328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,8191,0.037962667644023895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,8191,0.0366293340921402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,8191,0.04029866556326548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,8191,0.037231999138991036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,8191,0.0365226666132609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,8191,0.03735466549793879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,8191,0.03651199986537298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,16383,0.04073066761096319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,16383,0.04991999765237173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,16383,0.04549333453178406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,16383,0.04437333345413208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,16383,0.04673600196838379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,16383,0.04531733194986979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,16383,0.04458666841189066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,16383,0.03977066775163015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,16383,0.04711999992529551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,16383,0.05064000189304352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,16383,0.04502399762471517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,16383,0.04540266593297323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,16383,0.04487466812133789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,16383,0.044639999667803444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,32767,0.05365333457787832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,32767,0.06188266475995382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,32767,0.05354666709899902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,32767,0.056602666775385536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,32767,0.050186668833096824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,32767,0.04934933284918467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,32767,0.04906133313973745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,32767,0.053674668073654175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,32767,0.05820799867312113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,32767,0.053397332628568016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,32767,0.06205333272616068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,32767,0.051402668158213295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,32767,0.04986133178075155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,32767,0.04987200101216634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,65535,0.06856533388296764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,65535,0.07622933387756348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,65535,0.07203733424345653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,65535,0.0692853331565857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,65535,0.07030933101971944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,65535,0.06779199838638306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,65535,0.06753600140412648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,65535,0.06855466465155284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,65535,0.07655466596285503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,65535,0.07223466535409291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,65535,0.07017600039641063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,65535,0.06921599805355072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,65535,0.0686773310105006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,65535,0.06769066552321117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,131071,0.09690133730570476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,131071,0.10022399822870891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,131071,0.0939466655254364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,131071,0.09829333424568176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,131071,0.092357337474823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,131071,0.09226133426030476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,131071,0.09644800424575806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,131071,0.09150399764378865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,131071,0.10110933581988017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,131071,0.09571733077367146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,131071,0.09363733728726704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,131071,0.0925333301226298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,131071,0.09205866853396098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,131071,0.09181867043177287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,1,0.027813332776228588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,1,0.02719466636578242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,1,0.02677333354949951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,1,0.022656001150608063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,1,0.026501332720120747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,1,0.026309333741664886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,1,0.026464000344276428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,1,0.02788266787926356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,1,0.027082666754722595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,1,0.0227360005180041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,1,0.026591998835404713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,1,0.026309333741664886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,1,0.026757332185904186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,1,0.026554666459560394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,3,0.02272533377011617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,3,0.027445333699385326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,3,0.026357332865397137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,3,0.026762666801611584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,3,0.02677333354949951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,3,0.026149332523345947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,3,0.022618666291236877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,3,0.026341333985328674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,3,0.027530667682488758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,3,0.02714666724205017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,3,0.02610666553179423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,3,0.026560001075267792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,3,0.0264533335963885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,3,0.0262773334980011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,7,0.022485333184401195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,7,0.02741333345572154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,7,0.026954665780067444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,7,0.026485333840052288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,7,0.02606400102376938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,7,0.02605866640806198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,7,0.026165333886941273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,7,0.027327999472618103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,7,0.02250133454799652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,7,0.026752000053723652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,7,0.02651199946800868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,7,0.02601066728432973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,7,0.026186667382717133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,7,0.026127999027570088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,15,0.02184533327817917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,15,0.026416001220544178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,15,0.026015999416510265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,15,0.027482666075229645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,15,0.025674665967623394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,15,0.025850666066010792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,15,0.0258240004380544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,15,0.021882665654023487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,15,0.02719466636578242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,15,0.02645866572856903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,15,0.02585600068171819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,15,0.02586666742960612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,15,0.026154667139053345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,15,0.025909334421157837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,31,0.021344001094500225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,31,0.026895999908447266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,31,0.026176000634829204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,31,0.025759999950726826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,31,0.02536533276240031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,31,0.025455998877684276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,31,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,31,0.02678400029738744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,31,0.021754667162895203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,31,0.0262773334980011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,31,0.02571733295917511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,31,0.02548266698916753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,31,0.02536533276240031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,31,0.025424001117547352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,63,0.020517333100239437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,63,0.02532266577084859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,63,0.025802666942278545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,63,0.02475200096766154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,63,0.02426133304834366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,63,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,63,0.02438933402299881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,63,0.0206133338312308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,63,0.025765334566434223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,63,0.02475733309984207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,63,0.024330665667851765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,63,0.024698667228221893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,63,0.024506665766239166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,63,0.02447466552257538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,127,0.02367999901374181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,127,0.028789333999156952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,127,0.027610667049884796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,127,0.027429332335789997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,127,0.026789332429567974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,127,0.026757332185904186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,127,0.026501332720120747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,127,0.0236160010099411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,127,0.02890666574239731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,127,0.026741333305835724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,127,0.027488000690937042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,127,0.02739733209212621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,127,0.02657066782315572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,255,0.023647998770078022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,127,0.026421333352724712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,255,0.02903466671705246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,255,0.02777066578467687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,255,0.027130665878454845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,255,0.030368000268936157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,255,0.026416001220544178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,255,0.02898666759332021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,255,0.02678400029738744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,255,0.023765332996845245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,255,0.026922665536403656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,255,0.02712533374627431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,255,0.026506667335828144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,255,0.027823999524116516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,255,0.026474667092164356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,511,0.02606400102376938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,511,0.02881066749493281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,511,0.02994133283694585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,511,0.027989332874615986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,511,0.027386667827765148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,511,0.027552001178264618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,511,0.02743999908367793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,511,0.026047999660174053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,511,0.030048000315825146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,511,0.028549333413441975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,511,0.02802666773398717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,511,0.02741866558790207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,511,0.027434666951497395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,511,0.027621333797772724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,1023,0.02794133375088374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,1023,0.03290133426586787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,1023,0.029930666089057922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,1023,0.030933332939942677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,1023,0.029530666768550873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,1023,0.029466666281223297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,1023,0.029653333127498627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,1023,0.02809600035349528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,1023,0.03270400067170461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,1023,0.030917334059874218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,1023,0.02962133288383484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,1023,0.030063999195893604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,1023,0.029616000751654308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,1023,0.029370665550231934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,2047,0.029792000850041706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,2047,0.03604800005753835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,2047,0.03417599946260452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,2047,0.033344000577926636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,2047,0.032826667030652366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,2047,0.03292799989382426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,2047,0.03246400008598963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,2047,0.029680001238981884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,2047,0.03606933355331421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,2047,0.034160000582536064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,2047,0.03329066683848699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,2047,0.03297599901755651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,2047,0.032885332902272545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,2047,0.032511999209721885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,4095,0.03286933402220408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,4095,0.03643733263015747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,4095,0.03506666670242945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,4095,0.03786666691303253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,4095,0.03491200009981791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,4095,0.034634667138258614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,4095,0.03274133304754893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,4095,0.03806933263937632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,4095,0.03612266729275385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,4095,0.035445332527160645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,4095,0.034917332231998444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,4095,0.03506666670242945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,4095,0.035061334570248924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,4095,0.035445332527160645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,8191,0.038346665600935616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,8191,0.0432533323764801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,8191,0.039887999494870506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,8191,0.04074666649103165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,8191,0.040106666584809623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,8191,0.03965866565704346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,8191,0.03893866638342539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,8191,0.03870933254559835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,8191,0.043162668744723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,8191,0.04098666707674662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,8191,0.0403466671705246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,8191,0.04033066580692927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,8191,0.039893334110577904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,16383,0.05055999755859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,16383,0.0581226646900177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,8191,0.03904533386230469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,16383,0.05481066803137461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,16383,0.04890666902065277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,16383,0.04612799982229868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,16383,0.045824001232783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,16383,0.054560000697771706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,16383,0.04595733185609182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,16383,0.0513866643110911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,16383,0.057989334066708885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,16383,0.05180266499519348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,16383,0.04598399996757507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,16383,0.045381332437197365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,16383,0.04528533418973287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,32767,0.06649066507816315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,32767,0.07366399963696797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,32767,0.06867200136184692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,32767,0.06504533191521962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,32767,0.06533866624037425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,32767,0.06678399940331776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,32767,0.06611200173695882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,32767,0.06483733157316844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,32767,0.07358400026957194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,32767,0.06806933383146922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,32767,0.06680533289909363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,32767,0.06512000163396199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,32767,0.06437333424886067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,65535,0.09383466839790344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,65535,0.09841600060462952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,65535,0.0918880005677541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,32767,0.06551466882228851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,65535,0.0899679958820343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,65535,0.08920533458391826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,65535,0.08872000376383464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,65535,0.09451199571291606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,65535,0.08794666330019633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,65535,0.09686932961146037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,65535,0.08880533774693807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,65535,0.09271466732025146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,65535,0.09026666482289632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,65535,0.08898133039474487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,65535,0.08833600083986919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,131071,0.14629333217938742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,131071,0.14126933614412943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,131071,0.1506239970525106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,131071,0.13716800014177957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,131071,0.1388053297996521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,131071,0.13715199629465738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,131071,0.1364479959011078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,131071,0.1462399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,131071,0.1497706671555837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,131071,0.14148799578348795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,131071,0.13716800014177957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,131071,0.13959999879201254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,131071,0.1365013321240743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,131071,0.13620799779891968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,1,0.023002666731675465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,1,0.02829866607983907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,1,0.02737066646416982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,1,0.02312533309062322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,1,0.026858667532602947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,1,0.026730666557947796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,1,0.026698666314284008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,1,0.02826666583617528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,1,0.026928000152111053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,1,0.027280000348885853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,1,0.027119999130566914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,1,0.02696000039577484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,1,0.026698666314284008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,3,0.023039999107519787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,3,0.027888000011444092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,1,0.02664000044266383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,3,0.0268053337931633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,3,0.02736533433198929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,3,0.026799999177455902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,3,0.02658133457104365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,3,0.022783999641736347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,3,0.02722666660944621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,3,0.026709333062171936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,3,0.028031999866167705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,3,0.02678400029738744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,3,0.026335999369621277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,3,0.026565333207448322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,3,0.026554666459560394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,7,0.022853332261244457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,7,0.0278613343834877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,7,0.026613332331180573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,7,0.026554666459560394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,7,0.027109332382678986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,7,0.02276800076166789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,7,0.02638400097688039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,7,0.027877333263556164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,7,0.02649066597223282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,7,0.026858667532602947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,7,0.027050666511058807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,7,0.026261332134405773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,7,0.026389333109060924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,15,0.022570667167504627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,7,0.02644266684850057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,15,0.02759466568628947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,15,0.02700799951950709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,15,0.02630399912595749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,15,0.02610666553179423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,15,0.026213333010673523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,15,0.022389332453409832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,15,0.025989333788553875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,15,0.027658666173617046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,15,0.02641066660483678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,15,0.026133333643277485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,15,0.026122666895389557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,15,0.026821332673231762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,15,0.026149332523345947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,31,0.027215999861558277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,31,0.022069332500298817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,31,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,31,0.025653332471847534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,31,0.02622933437426885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,31,0.025749333202838898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,31,0.02550400048494339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,31,0.022197333474953968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,31,0.0271573339899381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,31,0.02647999922434489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,31,0.02593066543340683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,31,0.025850666066010792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,31,0.02606933315594991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,63,0.021162666380405426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,31,0.025392000873883564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,63,0.02608533451954524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,63,0.025450666745503742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,63,0.024682665864626568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,63,0.024869332710901897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,63,0.020874666670958202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,63,0.02475733309984207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,63,0.02470933397610982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,63,0.024714666108290356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,63,0.025493333737055462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,63,0.02610666553179423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,63,0.024864000578721363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,63,0.02437866727511088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,63,0.024842667082945507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,127,0.024138666689395905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,127,0.029125332832336426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,127,0.027845333019892376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,127,0.027488000690937042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,127,0.027189334233601887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,127,0.026965332527955372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,127,0.02681066592534383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,127,0.029215998947620392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,127,0.0278613343834877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,127,0.027621333797772724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,127,0.024197332561016083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,127,0.027210667729377747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,127,0.02701866626739502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,127,0.02685333291689555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,255,0.025706666211287182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,255,0.029834667841593426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,255,0.028501334289709728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,255,0.027999999622503918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,255,0.027290667096773785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,255,0.027658666173617046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,255,0.02720533311367035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,255,0.025706666211287182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,255,0.02977599948644638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,255,0.02740799884001414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,255,0.02788266787926356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,255,0.028501334289709728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,255,0.027215999861558277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,255,0.027674667537212372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,511,0.027888000011444092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,511,0.032698666055997215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,511,0.03067733347415924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,511,0.028864001234372456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,511,0.029861333469549816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,511,0.028079998989899952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,511,0.028912000358104706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,511,0.029333333174387615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,511,0.03276800115903219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,511,0.030618667602539062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,511,0.02945599953333537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,511,0.029450667401154835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,511,0.028890666862328846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,1023,0.029674666623274486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,1023,0.033615998923778534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,1023,0.03557866563399633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,1023,0.032416000962257385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,511,0.029205332199732464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,1023,0.0321066677570343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,1023,0.03166933357715607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,1023,0.03193599979082743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,1023,0.029365333418051403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,1023,0.033488000432650246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,1023,0.03221333275238673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,1023,0.03565866748491923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,1023,0.0315733328461647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,1023,0.0320266659061114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,1023,0.031957333286603294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,2047,0.033029332756996155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,2047,0.037178667883078255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,2047,0.035743998984495796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,2047,0.03435199956099192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,2047,0.033728001018365227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,2047,0.03403733422358831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,2047,0.03276800115903219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,2047,0.036858665446440377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,2047,0.035760000348091125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,2047,0.03403199960788091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,2047,0.03430933256944021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,2047,0.033733333150545754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,2047,0.03408533334732056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,2047,0.033615998923778534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,4095,0.03756800045569738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,4095,0.04213866591453552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,4095,0.03989866624275843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,4095,0.03870933254559835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,4095,0.03825599948565165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,4095,0.038106667498747505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,4095,0.0383146678407987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,4095,0.03814399987459183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,4095,0.04265599946180979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,4095,0.039093332986036934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,4095,0.039690665900707245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,4095,0.03847466657559077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,4095,0.038378665844599404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,4095,0.03818666686614355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,8191,0.051488002141316734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,8191,0.05427733560403188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,8191,0.0509493350982666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,8191,0.044778664906819664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,8191,0.04572266836961111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,8191,0.04435733457406362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,8191,0.044031997521718345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,8191,0.052469333012898765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,8191,0.05457599957784017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,8191,0.05054933329423269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,8191,0.0469706654548645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,8191,0.04438399771849314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,8191,0.04484266539414724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,16383,0.06539733211199443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,8191,0.04353066782156626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,16383,0.06622399886449178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,16383,0.07166933516661327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,16383,0.0647573322057724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,16383,0.06321066617965698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,16383,0.06320533156394958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,16383,0.06266666452089946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,16383,0.06599999964237213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,16383,0.07186666627724965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,16383,0.06723199784755707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,16383,0.06460266808668773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,16383,0.06348800162474315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,16383,0.06317866841952006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,16383,0.06258666515350342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,32767,0.09556800127029419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,32767,0.0935093363126119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,32767,0.09077866872151692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,32767,0.08804800113042195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,32767,0.08710400263468425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,32767,0.0867199997107188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,32767,0.086709330479304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,32767,0.09431999921798706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,32767,0.09027199943860371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,32767,0.09635200103123982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,32767,0.08710933725039165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,32767,0.08866666754086812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,32767,0.08714666962623596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,32767,0.08635733524958293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,65535,0.14994133512179056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,65535,0.14458133776982626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,65535,0.1402453382809957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,65535,0.1360640029112498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,65535,0.1358560025691986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,65535,0.13547733426094055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,65535,0.14457600315411887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,65535,0.15005333224932352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,65535,0.13481600085894266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,65535,0.13936533530553183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,65535,0.13562132914861044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,65535,0.13699199755986533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,65535,0.13550933202107748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,65535,0.1348426640033722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,131071,0.25757867097854614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,131071,0.24022400379180908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,131071,0.2323733369509379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,131071,0.2355519930521647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,131071,0.23176532983779907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,131071,0.23202667633692423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,131071,0.22951465845108032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,131071,0.25736000140508014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,131071,0.2408213416735331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,131071,0.23547732830047607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,131071,0.2315466602643331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,131071,0.23241066932678223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,131071,0.23059733708699545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,1,0.023898666103680927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,131071,0.22989867130915323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,1,0.02889599899450938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,1,0.02794666588306427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,1,0.02754666656255722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,1,0.027269333600997925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,1,0.027077332139015198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,1,0.026917333404223125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,1,0.024218666056791942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,1,0.02794666588306427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,1,0.02752533306678136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,1,0.02733866622050603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,1,0.027119999130566914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,1,0.028858666618665058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,3,0.023984000086784363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,1,0.02696000039577484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,3,0.028778667251269024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,3,0.026975999275843304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,3,0.02788266787926356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,3,0.027535999814669292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,3,0.05120000243186951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,3,0.0239680012067159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,3,0.026837334036827087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,3,0.0288426677385966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,3,0.027637332677841187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,3,0.027503999571005504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,3,0.02722666660944621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,3,0.026816000541051228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,3,0.02699200063943863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,7,0.028602667152881622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,7,0.02367466688156128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,7,0.027445333699385326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,7,0.027056001126766205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,7,0.026933332284291584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,7,0.026943999032179516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,7,0.028384000062942505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,7,0.023930666347344715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,7,0.04922133187452952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,7,0.027119999130566914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,7,0.027610667049884796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,7,0.026954665780067444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,7,0.026752000053723652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,15,0.023520000278949738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,7,0.03250666707754135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,15,0.02812266598145167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,15,0.027258666853109997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,15,0.02679466704527537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,15,0.02659733345111211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,15,0.02638400097688039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,15,0.02672533442576726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,15,0.028234665592511494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,15,0.023397333920001984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,15,0.027258666853109997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,15,0.026746665438016255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,15,0.026362667481104534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,15,0.026629333694775898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,15,0.026741333305835724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,31,0.02288000037272771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,31,0.02685333291689555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,31,0.027797333896160126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,31,0.026314665873845417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,31,0.02601066728432973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,31,0.02606400102376938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,31,0.02624533325433731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,31,0.027679999669392902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,31,0.027050666511058807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,31,0.025370667378107708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,31,0.02638400097688039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,31,0.026314665873845417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,63,0.021914665897687275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,31,0.026149332523345947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,63,0.025701334079106648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,63,0.02678400029738744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,31,0.02629866699377696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,63,0.025429333249727886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,63,0.025098666548728943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,63,0.025253333151340485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,63,0.023706667125225067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,63,0.02513599892457326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,63,0.025941332181294758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,63,0.02679466704527537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,63,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,63,0.025290665527184803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,63,0.024911999702453613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,127,0.026191999514897663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,127,0.030229332546393078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,63,0.024933333198229473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,127,0.028832000990708668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,127,0.028304000695546467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,127,0.0277813325325648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,127,0.02788266787926356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,127,0.0277813325325648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,127,0.02619733413060506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,127,0.028858666618665058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,127,0.029882666965325672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,127,0.0283146674434344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,127,0.027717334528764088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,127,0.027962667246659596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,127,0.02749866743882497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,255,0.027834666272004444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,255,0.032618666688601174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,255,0.029909332593282063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,255,0.02923733244339625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,255,0.029317334294319153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,255,0.02887466549873352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,255,0.030602666238943737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,255,0.027776000400384266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,255,0.0295413335164388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,255,0.032645332316557564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,255,0.029631999631722767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,255,0.030821333328882854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,255,0.02888533224662145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,511,0.029887999097506206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,255,0.028922667105992634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,511,0.03505599995454153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,511,0.03127466638882955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,511,0.03234133372704188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,511,0.03350399931271871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,511,0.031258667508761086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,511,0.03514666606982549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,511,0.031770666440327965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,511,0.03050133337577184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,511,0.03320533285538355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,511,0.03146666785081228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,511,0.03232000023126602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,511,0.031888000667095184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,1023,0.038431999584039055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,511,0.03139200061559677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,1023,0.035088000198205314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,1023,0.03324266771475474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,1023,0.03449599941571554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,1023,0.03440533330043157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,1023,0.03755199909210205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,1023,0.0341386670867602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,1023,0.033471999069054924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,1023,0.03587199995915095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,1023,0.03385599950949351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,1023,0.033957332372665405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,1023,0.0336053321758906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,1023,0.03435733417669932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,2047,0.04002666721741358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,1023,0.033530667424201965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,2047,0.04301333427429199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,2047,0.03851733356714249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,2047,0.038160001238187156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,2047,0.03821333249409994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,2047,0.03812800099452337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,2047,0.03914133210976919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,2047,0.04306666553020477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,2047,0.040063999593257904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,2047,0.03904533386230469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,2047,0.0383093332250913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,2047,0.03854399919509888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,2047,0.03806933263937632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,4095,0.05212266743183136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,4095,0.05450133482615153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,4095,0.04548799991607666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,4095,0.05035200218359629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,2047,0.03749333322048187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,4095,0.04451733330885569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,4095,0.04378666480382284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,4095,0.05402133365472158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,4095,0.04324266811211904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,4095,0.05207466582457224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,4095,0.05008533100287119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,4095,0.042447999119758606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,4095,0.044079999128977455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,4095,0.044938668608665466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,4095,0.04350399971008301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,8191,0.06814399858315785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,8191,0.06665066878000896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,8191,0.07153066496054332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,8191,0.06433600187301636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,8191,0.06373333434263866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,8191,0.06233066817124685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,8191,0.07153066496054332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,8191,0.06799999872843425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,8191,0.06258666515350342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,8191,0.0664213349421819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,8191,0.0643093337615331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,8191,0.0631466656923294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,8191,0.06324266890684764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,8191,0.06221333146095276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,16383,0.09398933251698811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,16383,0.08985066413879395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,16383,0.08802133798599243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,16383,0.08611200253168742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,16383,0.09618133306503296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,16383,0.08637866377830505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,16383,0.0858133335908254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,16383,0.0960693359375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,16383,0.09186666210492452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,16383,0.09619733691215515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,16383,0.08749866485595703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,16383,0.08628799517949422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,16383,0.08657067020734151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,16383,0.08551466464996338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,32767,0.1469386617342631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,32767,0.15011733770370483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,32767,0.1390666663646698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,32767,0.1357973317305247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,32767,0.13487466176350912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,32767,0.1334933340549469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,32767,0.1346453328927358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,32767,0.15000533064206442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,32767,0.1453013320763906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,32767,0.135754664738973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,32767,0.13486933708190918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,32767,0.13912000258763632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,32767,0.13424000144004822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,65535,0.25759466489156085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,65535,0.24057066440582275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,65535,0.2344800035158793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,65535,0.23066665728886923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,32767,0.13329600294431052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,65535,0.22949333985646567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,65535,0.22874132792154947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,65535,0.22989332675933838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,65535,0.2341973384221395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,65535,0.24133867025375366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,65535,0.22985066970189413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,65535,0.2589813272158305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,65535,0.22857600450515747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,65535,0.22910932699839273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,65535,0.23069334030151367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,131071,0.474239985148112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,131071,0.42951468626658124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,131071,0.42292265097300213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,131071,0.41992000738779706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,131071,0.41786134243011475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,131071,0.42109866937001544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,131071,0.47518400351206463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,131071,0.4285866816838582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,131071,0.41656001408894855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,131071,0.41759999593098956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,131071,0.42282132307688397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,131071,0.41947734355926514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,131071,0.42159998416900635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,131071,0.41624534130096436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,1,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,1,0.028938665986061096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,1,0.02861333390076955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,1,0.02829866607983907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,1,0.02812266598145167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,1,0.030037333567937214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,1,0.028309332827727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,1,0.026522666215896606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,1,0.029930666089057922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,1,0.028592000404993694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,1,0.02905600021282832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,1,0.027973333994547527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,1,0.028304000695546467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,3,0.02625600000222524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,1,0.02810666710138321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,3,0.029909332593282063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,3,0.0278613343834877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,3,0.02899733434120814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,3,0.02810666710138321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,3,0.028373333315054577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,3,0.028138667345046997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,3,0.02980799973011017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,3,0.02869333326816559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,3,0.02790933350721995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,3,0.028442665934562683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,3,0.02621866762638092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,3,0.028175999720891316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,7,0.02628266563018163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,3,0.02805333336194356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,7,0.02976000060637792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,7,0.028416000306606293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,7,0.028309332827727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,7,0.027776000400384266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,7,0.027877333263556164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,7,0.027797333896160126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,7,0.029663999875386555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,7,0.026186667382717133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,7,0.02850666642189026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,7,0.02829866607983907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,7,0.0278613343834877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,15,0.02569066733121872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,7,0.027823999524116516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,15,0.029818666477998097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,15,0.028592000404993694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,7,0.02773333340883255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,15,0.0278613343834877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,15,0.027445333699385326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,15,0.02770666778087616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,15,0.02937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,15,0.02603733291228612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,15,0.027535999814669292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,15,0.027866666515668232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,15,0.027600000301996868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,15,0.02754666656255722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,15,0.02842666705449422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,15,0.027424000203609467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,31,0.025493333737055462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,31,0.02906133234500885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,31,0.02808533360560735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,31,0.027509334186712902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,31,0.02720533311367035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,31,0.027114666998386383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,31,0.02550933261712392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,31,0.02916266769170761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,31,0.027290667096773785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,31,0.02792533238728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,31,0.027290667096773785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,31,0.027221334477265675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,31,0.02717333287000656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,31,0.02701333413521449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,63,0.02515733242034912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,63,0.027045334378878277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,63,0.02622933437426885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,63,0.02792000025510788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,63,0.02609066665172577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,63,0.02624000112215678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,63,0.025989333788553875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,63,0.027855999767780304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,63,0.026842666169007618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,63,0.02465066562096278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,63,0.026261332134405773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,63,0.025941332181294758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,63,0.0260959987839063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,63,0.026234666506449383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,127,0.026629333694775898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,127,0.033413333197434746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,127,0.03036266565322876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,127,0.029829333225886028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,127,0.03158933420976003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,127,0.029872000217437744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,127,0.033301333586374916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,127,0.026858667532602947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,127,0.029743999242782593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,127,0.031034665803114574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,127,0.02998399982849757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,127,0.029989334444204967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,127,0.030565333863099415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,127,0.029578665892283123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,255,0.03288000077009201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,255,0.03215466688076655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,255,0.03284800052642822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,255,0.0317493329445521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,255,0.03572800010442734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,255,0.03173333406448364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,255,0.03177600105603536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,255,0.03277866790692011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,255,0.03500800083080927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,255,0.03175999969244003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,255,0.032858667274316154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,255,0.031871999303499855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,255,0.03176533430814743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,511,0.035317334036032356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,511,0.03871466716130575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,255,0.03207999964555105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,511,0.03522666543722153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,511,0.03472533325354258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,511,0.03401066611210505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,511,0.03422933320204417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,511,0.033930666744709015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,511,0.03563733398914337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,511,0.03526400029659271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,511,0.03405333310365677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,511,0.03862400104602178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,511,0.033770665526390076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,511,0.03362133353948593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,1023,0.042453333735466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,511,0.03417066733042399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,1023,0.04407466451327006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,1023,0.04043200115362803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,1023,0.03905066599448522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,1023,0.03875733415285746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,1023,0.038405333956082664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,1023,0.03909866760174433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,1023,0.042463997999827065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,1023,0.04371733466784159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,1023,0.03984533250331879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,1023,0.038880000511805214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,1023,0.038906666139761605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,1023,0.03860266755024592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,2047,0.05612266560395559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,2047,0.05542399982611338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,2047,0.05158400038878123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,2047,0.047322665651639305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,1023,0.03876800090074539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,2047,0.04481600224971771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,2047,0.04558933277924856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,2047,0.0435146689414978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,2047,0.054917335510253906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,2047,0.05619200070699056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,2047,0.05077333251635233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,2047,0.04530133306980133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,2047,0.04321066538492838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,2047,0.04769066472848257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,2047,0.04409066836039225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,4095,0.0730506678422292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,4095,0.07006399830182393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,4095,0.06687999765078227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,4095,0.06364266574382782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,4095,0.06457599997520447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,4095,0.06980266670385997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,4095,0.06278933087984721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,4095,0.06728533407052358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,4095,0.07225599884986877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,4095,0.06279466549555461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,4095,0.06457066535949707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,4095,0.06293866535027821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,8191,0.09841600060462952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,4095,0.06371733546257019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,8191,0.09962133566538493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,4095,0.0629066675901413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,8191,0.08953600128491719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,8191,0.0881760021050771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,8191,0.09266666571299235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,8191,0.0879146655400594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,8191,0.08650133013725281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,8191,0.09828266501426697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,8191,0.09312533338864644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,8191,0.09910933176676433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,8191,0.09051199754079182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,8191,0.08742933471997578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,8191,0.08691199620564778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,8191,0.08849599957466125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,16383,0.1514400045077006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,16383,0.14773866534233093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,16383,0.13556266824404398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,16383,0.13467199603716531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,16383,0.13390933473904928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,16383,0.13967999815940857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,16383,0.13301333785057068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,16383,0.15180800358454385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,16383,0.14803199966748556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,16383,0.13994666934013367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,16383,0.13597333431243896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,16383,0.13498666882514954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,16383,0.13407466808954874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,32767,0.26450665791829425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,16383,0.13288000226020813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,32767,0.24537599086761475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,32767,0.23542400201161703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,32767,0.23171732823053995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,32767,0.2303839921951294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,32767,0.22766399383544922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,32767,0.24562132358551025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,32767,0.22893865903218588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,32767,0.23497599363327026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,32767,0.2307200034459432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,32767,0.26477332909901935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,32767,0.23043199380238852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,32767,0.22900267442067465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,32767,0.22937599817911783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,65535,0.4317013422648112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,65535,0.42445866266886395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,65535,0.4803200165430705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,65535,0.4183359940846761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,65535,0.41948266824086505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,65535,0.4161866505940755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,65535,0.418501337369283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,65535,0.4790773391723633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,65535,0.43142398198445636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,65535,0.41914133230845135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,65535,0.425653338432312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,65535,0.41608532269795734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,65535,0.4172746737798055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,65535,0.41970133781433105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,131071,0.9097867012023926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,131071,0.7987786928812662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,131071,0.8064853350321451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,131071,0.7926826477050781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,131071,0.7995253403981527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,131071,0.7956106662750244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,131071,0.7921973069508871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,131071,0.9103573163350424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,131071,0.8075573444366455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,131071,0.8002026875813802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,131071,0.7954826354980469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,131071,0.7998613516489664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,1,0.029530666768550873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,131071,0.7911413510640463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,131071,0.7904213269551595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,1,0.03355200091997782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,1,0.03139200061559677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,1,0.030975999931494396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,1,0.030394665896892548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,1,0.030479999879995983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,1,0.03333866596221924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,1,0.03084266682465871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,1,0.03129599988460541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,1,0.030559999247392017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,1,0.02972800036271413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,1,0.0305226668715477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,1,0.03046400099992752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,1,0.03053866575161616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,3,0.029648000995318096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,3,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,3,0.030389333764712017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,3,0.030165334542592365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,3,0.031167998909950256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,3,0.03166933357715607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,3,0.030495998760064442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,3,0.029792000850041706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,3,0.03163733333349228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,3,0.033386667569478355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,3,0.03053866575161616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,3,0.030453334252039593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,3,0.030410667260487873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,3,0.03049066662788391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,7,0.029391999046007793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,7,0.03311466674009959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,7,0.030458666384220123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,7,0.031354665756225586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,7,0.03048533449570338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,7,0.03027733415365219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,7,0.02958933264017105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,7,0.033013333876927696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,7,0.030159999926884968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,7,0.03162133445342382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,7,0.03054400036732356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,7,0.0306986669699351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,7,0.030266667405764263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,15,0.028954667349656422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,15,0.03068800022204717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,7,0.03035733352104823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,15,0.0329066663980484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,15,0.03017599880695343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,15,0.030192000170548756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,15,0.029813334345817566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,15,0.02977066735426585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,15,0.028815999627113342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,15,0.03088533381621043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,15,0.030293333033720653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,15,0.03294399877389272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,15,0.03013866643110911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,15,0.029824001093705494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,15,0.02961066613594691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,31,0.028373333315054577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,31,0.030767999589443207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,31,0.03239466746648153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,31,0.029450667401154835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,31,0.029290666182835896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,31,0.02945599953333537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,31,0.029872000217437744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,31,0.028565332293510437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,31,0.03212266663710276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,31,0.029861333469549816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,31,0.029317334294319153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,31,0.031136001149813335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,31,0.029445332785447437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,31,0.029391999046007793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,63,0.028005334238211315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,63,0.03196800003449122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,63,0.029887999097506206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,63,0.028858666618665058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,63,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,63,0.028250666956106823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,63,0.02779199928045273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,63,0.032101333141326904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,63,0.02845333268245061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,63,0.029951999584833782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,63,0.028965334097544353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,63,0.02845866729815801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,63,0.028213332096735638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,127,0.03010133405526479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,63,0.028202667832374573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,127,0.03461866577466329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,127,0.03146666785081228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,127,0.03234133372704188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,127,0.031311998764673867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,127,0.030970667799313862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,127,0.03086400032043457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,127,0.03238933285077413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,127,0.030080000559488933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,127,0.03142400085926056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,127,0.0313226655125618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,127,0.03453333427508672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,127,0.030896000564098358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,127,0.030847998956839245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,255,0.03362133353948593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,255,0.03714666763941447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,255,0.037087999284267426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,255,0.04146666576464971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,255,0.03664000084002813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,255,0.037258667250474296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,255,0.037989333271980286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,255,0.03356266766786575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,255,0.041434665520985924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,255,0.03806400050719579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,255,0.03696533292531967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,255,0.037061333656311035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,511,0.0495306650797526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,255,0.03682666768630346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,511,0.04475200176239014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,255,0.03678400069475174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,511,0.03997333347797394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,511,0.03928533444801966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,511,0.038805333276589714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,511,0.03885333240032196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,511,0.050111999114354454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,511,0.04604800045490265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,511,0.03995733211437861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,511,0.03867200016975403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,511,0.03896533449490865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,511,0.03973866750796636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,511,0.03878933439652125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,1023,0.05789333085219065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,511,0.03887466589609782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,1023,0.06334400177001953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,1023,0.04734933376312256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,1023,0.05216533442338308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,1023,0.046725332736968994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,1023,0.04494399825731913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,1023,0.04659733176231384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,1023,0.057989334066708885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,1023,0.0628053347269694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,1023,0.05228800078233083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,1023,0.044624000787734985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,1023,0.04660266637802124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,1023,0.04524266719818115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,2047,0.07670400043328603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,1023,0.04874666531880697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,2047,0.0680159976085027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,2047,0.07408533493677776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,2047,0.06483200192451477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,2047,0.06413333117961884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,2047,0.06314133107662201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,2047,0.06313600142796834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,2047,0.07448000212510426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,2047,0.06814399858315785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,2047,0.07584000130494435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,2047,0.06530666848023732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,2047,0.06421866516272227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,2047,0.06298133234182994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,2047,0.0629120022058487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,4095,0.10449066758155823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,4095,0.08930133779843648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,4095,0.087909330924352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,4095,0.09896533687909444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,4095,0.08741333087285359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,4095,0.09197333455085754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,4095,0.08668800195058186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,4095,0.10293866197268169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,4095,0.09866666793823242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,4095,0.09177600344022115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,4095,0.08754666646321614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,4095,0.08872532844543457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,4095,0.08710400263468425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,8191,0.15716800093650818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,4095,0.08690133690834045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,8191,0.14667200048764548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,8191,0.14017599821090698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,8191,0.136762668689092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,8191,0.13530666629473367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,8191,0.1349440018335978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,8191,0.13467199603716531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,8191,0.14648000399271646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,8191,0.14035200079282126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,8191,0.13713600238164267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,8191,0.1574079990386963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,8191,0.1345919966697693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,8191,0.13569600383440653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,8191,0.13491732875506082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,16383,0.2442773381868998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,16383,0.2734453280766805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,16383,0.2350506583849589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,16383,0.230186661084493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,16383,0.2286720077196757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,16383,0.23194666703542074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,16383,0.22918933629989624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,16383,0.27287999788920086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,16383,0.24354666471481323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,16383,0.23562665780385336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,16383,0.23246934016545615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,16383,0.22987200816472372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,16383,0.2290453314781189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,16383,0.23011734088261923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,32767,0.44042134284973145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,32767,0.4886026779810588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,32767,0.4292106628417969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,32767,0.4219199816385905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,32767,0.4187733332316081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,32767,0.41807464758555096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,32767,0.4882719914118449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,32767,0.4391466776529948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,32767,0.4288800160090129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,32767,0.4180533488591512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,32767,0.4230080048243205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,32767,0.4201546510060628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,32767,0.41787731647491455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,32767,0.4208693504333496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,65535,0.9208373228708903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,65535,0.815178632736206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,65535,0.7956319650014242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,65535,0.7929173310597738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,65535,0.8027040163675944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,65535,0.7990612983703613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,65535,0.7922879854838053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,65535,0.9203039805094401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,65535,0.8148907025655111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,65535,0.803978681564331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,65535,0.7957013448079427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,65535,0.7949279944101969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,65535,0.7913226286570231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,65535,0.8006079991658529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,131071,1.7841332753499348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,131071,1.5635414123535156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,131071,1.549722671508789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,131071,1.5452052752176921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,131071,1.5559093157450359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,131071,1.5415040651957195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,131071,1.5428479512532551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,131071,1.7835146586100261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,131071,1.5620053609212239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,131071,1.5498773256937664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,131071,1.5462239583333333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,1,0.03677866607904434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,131071,1.5611999829610188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,1,0.03879466652870178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,131071,1.539738655090332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,1,0.03654933224121729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,1,0.03483733286460241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,131071,1.5400746663411458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,1,0.03491200009981791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,1,0.034645333886146545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,1,0.0346666673819224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,1,0.03659199923276901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,1,0.03590933233499527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,1,0.038949333131313324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,1,0.034917332231998444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,1,0.03459733227888743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,1,0.03430933256944021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,1,0.03507733345031738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,3,0.03897066662708918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,3,0.04218666752179464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,3,0.0363520011305809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,3,0.03527999917666117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,3,0.034661332766215004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,3,0.034154665966828666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,3,0.0364479993780454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,3,0.03899200012286504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,3,0.03639466563860575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,3,0.03421333432197571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,3,0.03526933242877325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,3,0.034527999659379326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,3,0.03425066669782003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,3,0.03419200082619985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,7,0.0364479993780454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,7,0.03870933254559835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,7,0.03514666606982549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,7,0.034373333056767784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,7,0.03605866680542628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,7,0.034058667719364166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,7,0.03402133285999298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,7,0.036389333506425224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,7,0.0388373335202535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,7,0.03622400015592575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,7,0.035018667578697205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,7,0.03400533397992452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,7,0.0341333324710528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,7,0.03405333310365677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,15,0.03844800094763438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,15,0.03606933355331421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,15,0.03568533311287562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,15,0.03454400102297465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,15,0.0337119996547699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,15,0.03387200087308884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,15,0.036133334040641785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,15,0.03854399919509888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,15,0.03562133262554804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,15,0.03438399980465571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,15,0.03364799916744232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,15,0.03387200087308884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,15,0.03377600014209747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,15,0.03387733300526937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,31,0.03569599986076355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,31,0.03793066740036011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,31,0.03513599932193756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,31,0.03346133232116699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,31,0.03314133236805598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,31,0.04053333401679993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,31,0.033439998825391136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,31,0.03783999880154928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,31,0.035642666121323906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,31,0.03515733281771342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,31,0.033957332372665405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,31,0.03323200096686681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,31,0.03333866596221924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,31,0.03339199970165888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,63,0.03295466552178065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,63,0.03762666632731756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,63,0.034671999514102936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,63,0.032405334214369454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,63,0.03513599932193756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,63,0.03230933348337809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,63,0.032373333970705666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,63,0.03505599995454153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,63,0.03762666632731756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,63,0.034815999368826546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,63,0.03294933338960012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,63,0.03266666581233343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,63,0.03204799940188726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,63,0.0323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,127,0.03709333389997482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,127,0.04051200052102407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,127,0.03728533287843069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,127,0.0354666660229365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,127,0.03514133393764496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,127,0.03513066718975703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,127,0.03728533287843069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,127,0.04030933231115341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,127,0.035455999275048576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,127,0.03478399912516276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,127,0.03742400060097376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,127,0.03522133330504099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,127,0.035029334326585136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,127,0.03498133271932602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,255,0.04186666508515676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,255,0.039135999977588654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,255,0.044351999958356224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,255,0.03884266565243403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,255,0.03868266691764196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,255,0.04092800120512644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,255,0.038480001191298165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,255,0.04347200194994608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,255,0.04179200033346812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,255,0.041050667564074196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,255,0.03930133332808813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,255,0.03887466589609782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,255,0.03854399919509888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,511,0.06611200173695882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,511,0.05387733379999796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,511,0.058042665322621666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,255,0.0395359992980957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,511,0.05273599922657013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,511,0.05124799907207489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,511,0.04956800242265066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,511,0.04970133304595947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,511,0.05393599967161814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,511,0.05051200091838837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,511,0.05232533315817515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,511,0.058058664202690125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,511,0.0659093310435613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,511,0.04955733319123586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,511,0.05049066742261251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,1023,0.06932266553243001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,1023,0.08494399984677632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,1023,0.06770666440327962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,1023,0.07480533421039581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,1023,0.06678399940331776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,1023,0.06922666728496552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,1023,0.0664106657107671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,1023,0.06925333539644878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,1023,0.07467199862003326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,1023,0.06856533388296764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,1023,0.06728533407052358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,1023,0.08530132969220479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,1023,0.06602666775385539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,1023,0.0668746680021286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,2047,0.1063253382841746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,2047,0.09597333272298177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,2047,0.0981119970480601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,2047,0.0935040016969045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,2047,0.09156266848246257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,2047,0.090421328941981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,2047,0.09692266583442688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,2047,0.10712533195813496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,2047,0.09828266501426697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,2047,0.09277333815892537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,2047,0.09107200304667155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,2047,0.0923413336277008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,2047,0.09057066837946574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,2047,0.09076266487439473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,4095,0.15409066279729208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,4095,0.15189866224924722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,4095,0.14365866780281067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,4095,0.13795199990272522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,4095,0.1400213340918223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,4095,0.1390880048274994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,4095,0.13797866304715475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,4095,0.14416533708572388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,4095,0.15203733245531717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,4095,0.15422399838765463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,4095,0.14045866330464682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,4095,0.13872533043225607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,4095,0.13854933778444925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,8191,0.2503733237584432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,4095,0.13890666762987772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,8191,0.2402026653289795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,8191,0.26295467217763263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,8191,0.23305600881576538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,8191,0.235807995001475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,8191,0.2330026626586914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,8191,0.2325813372929891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,8191,0.2510506709416707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,8191,0.2390986680984497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,8191,0.2346186637878418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,8191,0.26340800523757935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,8191,0.234117329120636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,8191,0.23254932959874472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,8191,0.23157866795857748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,16383,0.5106720129648844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,16383,0.4368480046590169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,16383,0.45642133553822833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,16383,0.42417601744333905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,16383,0.4219839970270793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,16383,0.427839994430542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,16383,0.42122666041056317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,16383,0.5105173190434774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,16383,0.45398934682210285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,16383,0.4283786614735921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,16383,0.4240533510843913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,16383,0.44071467717488605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,16383,0.4206506808598836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,16383,0.42213332653045654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,32767,0.9383893013000488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,32767,0.8347146511077881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,32767,0.8148693243662516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,32767,0.8025866349538168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,32767,0.7959360281626383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,32767,0.8035786946614584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,32767,0.7982133229573568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,32767,0.8319573402404785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,32767,0.9386346340179443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,32767,0.8155999978383383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,32767,0.802079995473226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,32767,0.7985066572825114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,32767,0.8013652960459391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,32767,0.7942346731821696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,65535,1.8022079467773438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,65535,1.5801760355631511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,65535,1.561845302581787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,65535,1.550704002380371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,65535,1.5463147163391113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,65535,1.5594453811645508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,65535,1.8010932604471843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,65535,1.5822933514912922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,65535,1.562405268351237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,65535,1.550704002380371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,65535,1.5420106252034504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,65535,1.5472853978474934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,65535,1.556437333424886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,65535,1.544688065846761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,131071,3.524229367574056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,131071,3.0576585133870444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,131071,3.0768054326375327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,131071,3.0475571950276694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,131071,3.0390825271606445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,131071,3.071125348409017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,131071,3.0339199701944985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,131071,3.5233119328816733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,131071,3.055312156677246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,131071,3.077861467997233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,131071,3.0375467936197915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,131071,3.0452372233072915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,1,0.04484266539414724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,1,0.049029335379600525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,1,0.05420266588528951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,131071,3.0359360376993814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,131071,3.0697971979777017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,1,0.04154666761557261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,1,0.042965332667032875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,1,0.04166933397452036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,1,0.04196799794832865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,1,0.05427733560403188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,1,0.049029335379600525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,1,0.04297066728274027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,1,0.04196266829967499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,1,0.04470933477083842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,1,0.041738669077555336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,1,0.04147200038035711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,3,0.05464000006516775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,3,0.048842668533325195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,3,0.04497066636880239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,3,0.04269866645336151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,3,0.0418453315893809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,3,0.041589332123597465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,3,0.05442666510740916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,3,0.04491200049718221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,3,0.04881600042184194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,3,0.04137066751718521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,3,0.041573333243529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,3,0.04271999994913737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,3,0.04161600023508072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,3,0.041493333876132965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,7,0.0487413356701533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,7,0.05467733244101206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,7,0.04161066561937332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,7,0.04229333500067393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,7,0.044437333941459656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,7,0.0414986660083135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,7,0.04108799993991852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,7,0.048698668678601585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,7,0.05431999762852987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,7,0.04459733267625173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,7,0.042394667863845825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,7,0.0413973331451416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,7,0.041189332803090416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,7,0.04162666698296865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,15,0.04829333225886027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,15,0.04426133135954539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,15,0.0545653353134791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,15,0.041850666205088295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,15,0.04099733382463455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,15,0.041290665666262306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,15,0.04084266722202301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,15,0.05375466744105021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,15,0.04188799858093262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,15,0.044394666949907936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,15,0.04830400149027506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,15,0.04125866790612539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,15,0.04111466556787491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,15,0.040933333337306976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,31,0.05615466833114624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,31,0.04780266682306925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,31,0.043765331308046974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,31,0.04180799921353658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,31,0.04065066576004028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,31,0.04070399949947993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,31,0.05587733288606008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,31,0.04775466521581014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,31,0.04050666590531667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,31,0.043749332427978516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,31,0.041759997606277466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,31,0.04068266600370407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,31,0.040805332362651825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,31,0.040378667414188385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,63,0.05865600208441416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,63,0.04331199824810028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,63,0.04099733382463455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,63,0.04013866682847341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,63,0.0397119993964831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,63,0.04809600114822388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,63,0.03952533255020777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,63,0.059290667374928795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,63,0.047914668917655945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,63,0.04331199824810028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,63,0.04101866732041041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,63,0.03998400022586187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,63,0.039461334546407066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,63,0.03958933303753535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,127,0.05204799771308899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,127,0.06364266574382782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,127,0.04636266827583313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,127,0.04404800136884054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,127,0.04320000112056732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,127,0.04287999868392944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,127,0.06357866525650024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,127,0.04273599882920583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,127,0.043920000394185386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,127,0.043280000487963356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,127,0.051557332277297974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,127,0.04618666569391886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,127,0.04284266630808512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,127,0.042730664213498436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,255,0.04780800143877665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,255,0.05171733101209005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,255,0.047413334250450134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,255,0.06371733546257019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,255,0.057162667314211525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,255,0.07077333331108093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,255,0.04780800143877665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,255,0.0635040005048116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,255,0.07019733389218648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,255,0.057333335280418396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,255,0.04849066833655039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,255,0.051813334226608276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,255,0.047541335225105286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,255,0.048101335763931274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,511,0.07709866762161255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,511,0.07115733126799266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,511,0.08545066912968953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,511,0.06780800223350525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,511,0.06582933167616527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,511,0.06646933158238728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,511,0.08568533261617024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,511,0.0773119976123174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,511,0.06756799916426341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,511,0.06663999954859416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,511,0.06529066463311513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,511,0.07055999835332234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,511,0.06592000027497609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,511,0.06539200246334076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,1023,0.11476266384124756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,1023,0.0951039989789327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,1023,0.1016373336315155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,1023,0.09130666653315227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,1023,0.08994133273760478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,1023,0.09146666526794434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,1023,0.0890773336092631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,1023,0.11513599753379822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,1023,0.09499200185139973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,1023,0.09194667140642802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,1023,0.08990400036176045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,1023,0.10152533650398254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,1023,0.08916266759236653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,1023,0.08981866637865703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,2047,0.14323733250300089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,2047,0.14995200435320535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,2047,0.16978132724761963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,2047,0.13984533150990805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,2047,0.13851199547449747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,2047,0.13709333539009094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,2047,0.13798933227856955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,2047,0.16970133781433105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,2047,0.15014933546384177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,2047,0.14339199662208557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,2047,0.13979199528694153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,2047,0.1378773351510366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,2047,0.13738133509953818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,4095,0.2810879945755005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,4095,0.23835732539494833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,4095,0.2344906727472941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,2047,0.13737600048383078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,4095,0.24523200591405234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,4095,0.233514666557312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,4095,0.23191465934117636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,4095,0.22989867130915323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,4095,0.2802720069885254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,4095,0.23323200146357217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,4095,0.24442134300867716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,4095,0.23414399226506552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,4095,0.232095996538798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,4095,0.23786133527755737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,4095,0.22995734214782715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,8191,0.4280960162480672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,8191,0.4337173302968343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,8191,0.5224586725234985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,8191,0.4238719940185547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,8191,0.4223039944966634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,8191,0.4171733458836873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,8191,0.4220159848531087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,8191,0.5204906860987345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,8191,0.4229546785354614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,8191,0.4339786767959595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,8191,0.42758933703104657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,8191,0.4245386521021525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,8191,0.41786666711171466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,8191,0.42154133319854736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,16383,0.9550879796346029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,16383,0.8298719724019369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,16383,0.8576266765594482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,16383,0.8155093193054199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,16383,0.8061707019805908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,16383,0.8051466941833496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,16383,0.7997013727823893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,16383,0.9538666407267252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,16383,0.8306453227996826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,16383,0.8153279622395834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,16383,0.8590026696523031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,16383,0.806277354558309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,16383,0.8069653511047363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,16383,0.8004960219065348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,32767,1.8160959879557292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,32767,1.5634719530741374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,32767,1.5814720789591472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,32767,1.5551892916361492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,32767,1.5618346532185872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,32767,1.6108853022257488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,32767,1.549349308013916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,32767,1.8182560602823894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,32767,1.6108105977376301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,32767,1.5688319206237793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,32767,1.5625492731730144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,32767,1.5498080253601074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,32767,1.5821760495503743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,32767,1.548949400583903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,65535,3.5386826197306314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,65535,3.1071465810139975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,65535,3.1080106099446616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,65535,3.057114601135254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,65535,3.046047846476237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,65535,3.0415306091308594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,65535,3.0391947428385415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,65535,3.5369812647501626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,65535,3.1044960021972656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,65535,3.1092373530069985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,65535,3.04529603322347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,65535,3.06002140045166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,65535,3.063568115234375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,65535,3.0407679875691733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,131071,6.042416254679362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,131071,6.093258539835612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,131071,6.031754811604817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,131071,6.025152206420898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,131071,6.978250503540039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,131071,6.134101231892903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,131071,6.026186625162761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,131071,6.982170740763347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,131071,6.087274551391602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,131071,6.106810887654622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,131071,6.040032068888347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,1,0.07017600039641063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,1,0.09342933694521587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,131071,6.062357584635417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,131071,6.024986902872722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,1,0.05761066575845083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,1,0.06089599927266439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,1,0.05596266686916351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,1,0.05499200026194254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,1,0.055061335364977516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,1,0.07037866612275441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,1,0.09317866961161296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,1,0.05729066828886668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,1,0.060933331648508705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,1,0.05594133337338766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,1,0.054842665791511536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,1,0.05547733108202616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,3,0.07019733389218648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,131071,6.022698720296224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,3,0.09258133172988892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,3,0.060933331648508705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,3,0.057274664441744484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,3,0.056032001972198486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,3,0.05533333122730255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,3,0.054714664816856384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,3,0.06067200005054474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,3,0.0697866678237915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,3,0.09205333391825359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,3,0.05576533575852712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,3,0.05513066550095876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,3,0.05449066559473673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,3,0.05726933479309082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,7,0.06061866879463196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,7,0.057114665706952415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,7,0.055493334929148354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,7,0.07029333213965099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,7,0.092549333969752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,7,0.05442666510740916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,7,0.054245332876841225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,7,0.07076799869537354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,7,0.060677334666252136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,7,0.05701333284378052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,7,0.055359999338785805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,7,0.09273067116737366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,7,0.05452266832192739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,7,0.05418133238951365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,15,0.05961066484451294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,15,0.05624000231424967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,15,0.07125333448251088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,15,0.053616002202034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,15,0.05462400118509928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,15,0.09265599648157756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,15,0.05333333214124044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,15,0.05598933498064677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,15,0.09252799550692241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,15,0.07146133482456207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,15,0.053802669048309326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,15,0.054602667689323425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,15,0.05981333553791046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,31,0.07365866502126057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,31,0.05877333382765452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,31,0.05500799914201101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,15,0.05338133374849955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,31,0.09359467029571533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,31,0.05300800005594889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,31,0.053632001082102455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,31,0.05247466762860616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,31,0.0537013312180837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,31,0.07361599802970886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,31,0.055071999629338585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,31,0.05293866495291392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,31,0.052527998884518944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,31,0.059077332417170204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,31,0.09307199716567993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,63,0.05426666637261709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,63,0.07782933115959167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,63,0.059903999169667564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,63,0.05223466455936432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,63,0.09614933530489604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,63,0.051455999414126076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,63,0.050842667619387306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,63,0.06020266811052958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,63,0.07762666543324788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,63,0.05393599967161814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,63,0.05230933427810669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,63,0.09593066573143005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,63,0.05072000126043955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,127,0.08277866741021474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,63,0.05114666620890299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,127,0.06589866677920024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,127,0.0718399981657664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,127,0.10512533783912659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,127,0.06151466568311056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,127,0.05735999842484792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,127,0.05658133327960968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,127,0.07218666871388753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,127,0.06523199876149495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,127,0.08302933474381764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,127,0.061434666315714516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,127,0.10431999961535136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,127,0.057706668972969055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,127,0.057002668579419456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,255,0.09549333651860555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,255,0.08507200082143147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,255,0.07914133369922638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,255,0.11839999755223592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,255,0.07541866600513458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,255,0.07658133407433827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,255,0.075162669022878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,255,0.0965280036131541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,255,0.07916800181070964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,255,0.07667733232180278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,255,0.11844266454378764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,255,0.08528000116348267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,255,0.07492266595363617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,255,0.07514666517575581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,511,0.12329066793123881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,511,0.14704533418019614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,511,0.11010666688283284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,511,0.10310399532318115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,511,0.10015466809272766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,511,0.09930666287740071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,511,0.09859200318654378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,511,0.12272000312805176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,511,0.1471466620763143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,511,0.10973333319028218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,511,0.0995360016822815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,511,0.10108799735705058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,511,0.10309867064158122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,511,0.09909333785374959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,1023,0.15094932913780212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,1023,0.2049600084622701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,1023,0.14794133106867471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,1023,0.17138133446375528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,1023,0.15747732917467752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,1023,0.14671466747919717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,1023,0.1456160048643748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,1023,0.1511840025583903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,1023,0.1718506614367167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,1023,0.20521066586176553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,1023,0.15781333049138388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,1023,0.14792533715566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,1023,0.14643733700116476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,1023,0.1458133359750112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,2047,0.24501866102218628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,2047,0.31561599175135296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,2047,0.252405325571696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,2047,0.2655093272527059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,2047,0.2416106661160787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,2047,0.24177600940068564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,2047,0.23993066946665445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,2047,0.26502933104832965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,2047,0.31617599725723267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,2047,0.25124800205230713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,2047,0.24177600940068564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,2047,0.2448319991429647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,2047,0.2413333257039388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,2047,0.2398293415705363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,4095,0.44150932629903156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,4095,0.43514664967854816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,4095,0.4540586471557617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,4095,0.5489493211110433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,4095,0.4315520127614339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,4095,0.43344000975290936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,4095,0.42952533562978107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,4095,0.4531466563542684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,4095,0.4345386823018392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,4095,0.43323731422424316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,4095,0.430458664894104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,4095,0.5492053429285685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,4095,0.44043731689453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,4095,0.428928017616272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,8191,0.8530346552530924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,8191,0.8755626678466797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,8191,0.8265439669291178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,8191,0.9862079620361328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,8191,0.8160906632741293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,8191,0.8087573051452637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,8191,0.8069279988606771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,8191,0.8743626276652018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,8191,0.8305226961771647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,8191,0.8437493642171224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,8191,0.8163626988728842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,8191,0.985637346903483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,8191,0.8089546362559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,8191,0.8065120379130045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,16383,1.5742400487263997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,16383,1.6321600278218586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,16383,1.59769074122111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,16383,1.5621120134989421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,16383,1.5812212626139324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,16383,1.8508052825927734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,16383,1.5549866358439128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,16383,1.5995999972025554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,16383,1.6345866521199544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,16383,1.8497653007507324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,16383,1.5635627110799153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,16383,1.5579466819763184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,16383,1.576533317565918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,16383,1.5566026369730632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,32767,3.1703465779622397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,32767,3.076512018839518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,32767,3.095818519592285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,32767,3.0578559239705405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,32767,3.5740159352620444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,32767,3.048858642578125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,32767,3.0518665313720703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,32767,3.097818692525228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,32767,3.1375732421875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,32767,3.0598665873209634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,32767,3.574037233988444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,32767,3.077866554260254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,32767,3.0507148106892905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,32767,3.0491679509480796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,65535,6.123968124389648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,65535,6.092165629069011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,65535,6.125450770060222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,65535,7.012250900268555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,65535,6.034767786661784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,65535,6.041274388631185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,65535,6.032437642415364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,65535,7.015461603800456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,65535,6.154842376708984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,65535,6.130847930908203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,65535,6.058597564697266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,65535,6.0425599416097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,65535,6.035258611043294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,1,0.16178666551907858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,1,0.08678932984670003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,1,0.09659733374913533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,1,0.08336533109347026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,1,0.11492799719174702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,1,0.08143466711044312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,1,0.08203733464082082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,1,0.09583466251691182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,1,0.08674133817354839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,1,0.11494933565457661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,1,0.08341333270072937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,1,0.16172266999880472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,1,0.08158933122952779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,1,0.0820000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,3,0.08646399776140849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,3,0.11422399679819743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,3,0.09494933485984802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,3,0.15914666652679443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,3,0.08345599969228108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,3,0.08171733220418294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,3,0.081194669008255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,3,0.08709866801897685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,3,0.09530133008956909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,3,0.1139413317044576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,3,0.1597760021686554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,3,0.08180800080299377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,3,0.08309866487979889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,3,0.08132799963156383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,7,0.11417599519093831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,7,0.08568533261617024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,7,0.09532266855239868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,7,0.1584213376045227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,7,0.08231466511885326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,7,0.08050666749477386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,7,0.08137066662311554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,7,0.1141813298066457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,7,0.08570133646329244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,7,0.0953653355439504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,7,0.15834133823712668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,7,0.08262399832407634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,7,0.08133333424727122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,7,0.08076266447703044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,15,0.09452266494433086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,15,0.084197332461675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,15,0.114138662815094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,65535,6.033141454060872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,15,0.08092266817887624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,15,0.07928533355395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,15,0.15854400396347046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,15,0.07863999903202057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,15,0.09423466523488362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,15,0.11349333326021831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,15,0.15837867061297098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,15,0.08409066994984944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,15,0.07867733140786488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,15,0.0805920014778773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,15,0.07921599845091502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,31,0.11615999539693196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,31,0.08341866731643677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,31,0.09544000029563904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,31,0.0788320004940033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,31,0.15973866979281107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,31,0.0772159993648529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,31,0.0765066643555959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,31,0.08328000207742055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,31,0.07855999966462453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,31,0.07720000048478444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,31,0.11637866497039795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,31,0.0955573320388794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,31,0.160261332988739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,31,0.07643199960390727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,63,0.12052800258000691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,63,0.07972800234953563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,63,0.08803733189900716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,63,0.16334399580955505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,63,0.0755680004755656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,63,0.08381866415341695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,63,0.09949333469072978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,63,0.10021332899729411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,63,0.16286933422088623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,63,0.12098667025566101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,63,0.07992533346017201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,63,0.08819199601809184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,63,0.07667199770609538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,63,0.0832426647345225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,127,0.09382933378219604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,127,0.11027733484903972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,127,0.0997279981772105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,127,0.09239466985066731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,127,0.13264000415802002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,127,0.18326934178670248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,127,0.09132799506187439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,127,0.11102933684984843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,127,0.13159466783205667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,127,0.09226133426030476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,127,0.09422933061917622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,127,0.1000373363494873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,127,0.09045867125193278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,127,0.1819360057512919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,255,0.12452266613642375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,255,0.1341759959856669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,255,0.15769066413243613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,255,0.11566399534543355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,255,0.11387733618418376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,255,0.11942933003107707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,255,0.21041599909464517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,255,0.12422399719556172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,255,0.13542399803797403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,255,0.15762666861216226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,255,0.21109867095947266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,255,0.1144426663716634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,255,0.11870400110880534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,255,0.11619200309117635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,511,0.2662186622619629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,511,0.16707199811935425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,511,0.16359466314315796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,511,0.1843199928601583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,511,0.20916267236073813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,511,0.1730239987373352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,511,0.16300800442695618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,511,0.17368000745773315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,511,0.16703999042510986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,511,0.18482667207717896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,511,0.16356266538302103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,511,0.16209600369135538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,511,0.2671839992205302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,511,0.20933334032694498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,1023,0.27934932708740234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,1023,0.26629867156346637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,1023,0.3046346704165141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,1023,0.25682133436203003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,1023,0.3832639853159587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,1023,0.25490667422612506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,1023,0.2606026728947957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,1023,0.2669386665026347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,1023,0.2606133421262105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,1023,0.28010666370391846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,1023,0.2571893334388733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,1023,0.30552534262339276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,1023,0.38355199495951336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,1023,0.25487999121348065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,2047,0.49431999524434406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,2047,0.4493653376897176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,2047,0.4706559975941976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,2047,0.6086879968643188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,2047,0.4558826684951782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,2047,0.44604265689849854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,2047,0.44436800479888916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,2047,0.47111467520395917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,2047,0.457205335299174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,2047,0.4495360056559245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,2047,0.4445546468098958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,2047,0.4928106864293416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,2047,0.44414401054382324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,2047,0.6092480023701986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,4095,0.8739360173543295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,4095,0.8530186812082926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,4095,0.9119040171305338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,4095,0.8385919729868571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,4095,0.8255946636199951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,4095,1.0517919858296711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,4095,0.8265759944915771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,4095,0.8757759730021158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,4095,0.9112959702809652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,4095,0.8551039695739746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,4095,0.838047981262207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,4095,0.8318186601003011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,4095,1.0533706347147624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,4095,0.8245226542154948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,8191,1.636965274810791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,8191,1.5990559260050456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,8191,1.6098186175028484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,8191,1.6835840543111165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,8191,1.5745387077331543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,8191,1.9184106190999348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,8191,1.5727732976277669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,8191,1.5893707275390625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,8191,1.6341652870178223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,8191,1.9179360071818035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,8191,1.5736319224039714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,8191,1.6079360644022624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,8191,1.6962720553080242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,8191,1.5719733238220215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,16383,3.1352640787760415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,16383,3.081653277079264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,16383,3.186645189921061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,16383,3.108346621195475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,16383,3.6415627797444663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,16383,3.100031852722168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,16383,3.067861239115397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,16383,3.643744150797526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,16383,3.1044321060180664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,16383,3.080986658732096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,16383,3.134357452392578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,16383,3.1865758895874023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,16383,3.1029494603474936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,16383,3.0638345082600913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,32767,6.121402740478516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,32767,7.087141036987305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,32767,6.190389633178711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,32767,6.087200164794922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,32767,6.066928227742513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,32767,6.127706527709961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,32767,6.048255920410156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,32767,6.189546585083008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,32767,6.086314519246419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,32767,6.124933242797852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,32767,7.086405436197917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,32767,6.124448140462239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,32767,6.0703786214192705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,1,0.14848533272743225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,1,0.1630773345629374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,1,0.19537067413330078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,1,0.29811733961105347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,1,0.13873066504796347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,1,0.1365386644999186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,1,0.13486400246620178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,1,0.16325333714485168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,1,0.1953279972076416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,1,0.14867200454076132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,1,0.13894400000572205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,32767,6.048720041910808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,1,0.13598933815956116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,1,0.29842134316762287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,1,0.13523200154304504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,3,0.16142400105794272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,3,0.1470026671886444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,3,0.13749333222707114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,3,0.19322667519251505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,3,0.13337066769599915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,3,0.29309866825739544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,3,0.13430399696032205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,3,0.14680533607800803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,3,0.13741866747538248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,3,0.16149333119392395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,3,0.1930933396021525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,3,0.13447999954223633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,3,0.13288000226020813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,3,0.29385600487391156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,7,0.14524799585342407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,7,0.13562132914861044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,7,0.16009066502253214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,7,0.19293866554896036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,7,0.13174933195114136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,7,0.29154666264851886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,7,0.13276799519856772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,7,0.16013866662979126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,7,0.14518933494885763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,7,0.1355893313884735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,7,0.19266666968663534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,7,0.1328000028928121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,7,0.13144532839457193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,7,0.29203200340270996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,15,0.1428053379058838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,15,0.1581546664237976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,15,0.13402666648228964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,15,0.19144533077875772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,15,0.12878400087356567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,15,0.12990400195121765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,15,0.29029866059621173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,15,0.14325333635012308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,15,0.15762133399645487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,15,0.13424000144004822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,15,0.192303995291392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,15,0.13033066193262735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,15,0.1292639970779419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,31,0.13959999879201254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,31,0.19593600432078043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,31,0.15711466471354166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,31,0.12822399536768594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,31,0.1307199994723002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,15,0.2895573377609253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,31,0.2908373276392619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,31,0.1264693339665731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,31,0.19566933314005533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,31,0.1265546679496765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,31,0.1400320033232371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,31,0.12845866878827414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,31,0.13099199533462524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,31,0.15753066539764404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,31,0.29121599594751996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,63,0.14269866545995077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,63,0.13226667046546936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,63,0.16190933187802634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,63,0.1295413374900818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,63,0.12664000193277994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,63,0.20742400487263998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,63,0.3009439905484517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,63,0.14245866735776266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,63,0.13264000415802002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,63,0.1295253336429596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,63,0.16210666298866272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,63,0.12704533338546753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,63,0.2982986569404602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,63,0.2074986696243286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,127,0.18569066127141318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,127,0.22963732481002808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,127,0.1637493371963501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,127,0.1532853345076243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,127,0.14803199966748556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,127,0.1442080040772756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,127,0.3420906861623128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,127,0.1524853308995565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,127,0.14672000209490457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,127,0.18658133347829184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,127,0.23054399092992148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,127,0.16361066699028015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,127,0.1448853313922882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,127,0.34409066041310626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,255,0.19606933991114298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,255,0.21125866969426474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,255,0.20127467314402261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,255,0.27795199553171795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,255,0.23312000433603922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,255,0.19235734144846597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,255,0.3963093360265096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,255,0.21201600631078085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,255,0.19559999306996664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,255,0.23327465852101645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,255,0.20218133926391602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,255,0.2771413326263428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,255,0.19217065970102945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,255,0.3951359987258911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,511,0.3325493335723877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,511,0.29733866453170776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,511,0.2903839945793152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,511,0.3110453287760417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,511,0.3787733316421509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,511,0.28595733642578125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,511,0.5066080093383789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,511,0.29688533147176105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,511,0.31038399537404376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,511,0.33299734195073444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,511,0.29038933912913006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,511,0.378490686416626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,511,0.28537599245707196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,511,0.5072319904963175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,1023,0.5252159833908081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,1023,0.4869600137074788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,1023,0.4781706730524699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,1023,0.5001226663589478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,1023,0.4734933376312256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,1023,0.573914647102356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,1023,0.7296319802602133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,1023,0.49989867210388184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,1023,0.524890661239624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,1023,0.4859093427658081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,1023,0.47335998217264813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,1023,0.5714666843414307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,1023,0.4783946673075358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,1023,0.7294720013936361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,2047,0.9285706679026285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,2047,0.8777173360188802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,2047,0.8932000001271566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,2047,0.9830826918284098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,2047,0.8529973030090332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,2047,1.1758986314137776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,2047,0.8602506319681803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,2047,0.8916052977244059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,2047,0.9254399935404459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,2047,0.8733119964599609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,2047,0.9895626703898112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,2047,0.8604426383972168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,2047,0.8517173131306967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,2047,1.1753119627634685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,4095,1.7000640233357747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,4095,1.6638879776000977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,4095,1.7715786298116047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,4095,1.6300320625305176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,4095,2.0537333488464355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,4095,1.6020959218343098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,4095,1.6143786112467449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,4095,1.695583979288737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,4095,1.6579467455546062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,4095,1.6284747123718262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,4095,1.7834773063659668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,4095,1.6128427187601726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,4095,1.599082628885905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,4095,2.0582879384358725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,8191,3.2073227564493814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,8191,3.1886558532714844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,8191,3.78873602549235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,8191,3.3017921447753906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,8191,3.0934826532999673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,8191,3.1238667170206704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,8191,3.110133488972982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,8191,3.203472137451172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,8191,3.3237921396891275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,8191,3.1194238662719727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,8191,3.7841386795043945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,8191,3.1573120752970376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,8191,3.110410690307617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,8191,3.0946985880533853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,16383,6.145423889160156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,16383,7.232293446858724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,16383,6.205557505289714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,16383,6.110191980997722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,16383,6.368480046590169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,16383,6.094282786051433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,16383,6.077119827270508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,16383,6.207583745320638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,16383,6.153509140014648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,16383,6.181061426798503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,16383,6.3059037526448565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,16383,6.0974985758463545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,16383,7.229024251302083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,16383,6.080725351969401
