framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,16,1,0,0.013445333888133367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,16,1,0,0.013397333522637686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,16,2,0,0.012800000607967377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,16,8,0,0.012693333129088083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,16,2,0,0.02460266649723053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,16,4,0,0.01301866645614306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,16,16,0,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,16,32,0,0.01320533330241839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,16,64,0,0.013258667041858038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,16,4,0,0.012901333471139273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,16,8,0,0.01724799970785777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,16,16,0,0.012154666086037954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,16,32,0,0.012026666353146235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,16,64,0,0.012063999970753988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,32,1,0,0.01358933374285698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,32,2,0,0.01310933381319046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,32,4,0,0.013338666409254074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,32,8,0,0.0124746672809124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,32,32,0,0.01201066623131434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,32,16,0,0.035930665830771126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,32,1,0,0.013450667262077332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,32,2,0,0.029765332738558452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,32,64,0,0.012250666817029318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,32,4,0,0.014287999520699183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,32,8,0,0.01303999995191892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,32,16,0,0.012741333494583765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,32,32,0,0.012645332763592402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,32,64,0,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,64,2,0,0.01470400020480156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,64,1,0,0.014479999740918478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,64,8,0,0.014074667046467463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,64,16,0,0.013397333522637686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,64,4,0,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,64,32,0,0.013429333766301474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,64,64,0,0.013674666484196981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,64,8,0,0.0138026662170887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,64,1,0,0.014943999548753103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,64,2,0,0.014042666802803675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,64,32,0,0.013440000514189402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,64,4,0,0.014229333649079004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,64,16,0,0.01350933313369751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,64,64,0,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,128,2,0,0.01807466646035512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,128,1,0,0.01637866720557213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,128,4,0,0.038762666285037994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,128,32,0,0.014682666709025701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,128,8,0,0.015376000354687372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,128,16,0,0.014864000181357065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,128,1,0,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,128,64,0,0.014933332800865173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,128,16,0,0.015157333264748255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,128,8,0,0.01534933348496755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,128,4,0,0.015173333386580149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,128,32,0,0.014767999450365702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,128,2,0,0.017711999515692394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,128,64,0,0.014858666807413101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,256,1,0,0.019882666567961376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,256,2,0,0.01773333301146825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,256,4,0,0.017317333569129307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,256,16,0,0.01611199975013733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,256,32,0,0.015829333414634068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,256,8,0,0.016330666840076447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,256,64,0,0.016016000260909397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,256,1,0,0.019626667102177937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,256,2,0,0.017616000026464462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,256,16,0,0.015850666910409927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,256,64,0,0.015919999529918034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,256,32,0,0.01590399940808614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,256,8,0,0.016730666160583496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,256,4,0,0.0183146670460701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,512,2,0,0.0220320001244545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,512,1,0,0.034304000437259674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,512,4,0,0.020981334149837494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,512,8,0,0.020853333175182343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,512,64,0,0.019509332875410717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,512,32,0,0.019621333728233974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,512,16,0,0.01974933346112569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,512,8,0,0.020746666938066483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,512,2,0,0.022245332598686218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,512,4,0,0.02089066555102666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,512,1,0,0.033717334270477295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,512,16,0,0.021957332889238994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,512,32,0,0.01995733380317688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,512,64,0,0.019786667078733444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,1024,16,0,0.027989332874615986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,1024,2,0,0.04146133363246918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,1024,32,0,0.0272533322374026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,1024,4,0,0.029504001140594482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,1024,8,0,0.028192001084486645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,1024,1,0,0.07727466523647308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,1024,64,0,0.02658133457104365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,1024,4,0,0.02932800104220708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,1024,2,0,0.04154666761557261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,1024,32,0,0.02720533311367035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,1024,1,0,0.07601599891980489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,1024,8,0,0.028309332827727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,1024,64,0,0.026591998835404713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,1024,16,0,0.027802666028340656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,1536,4,0,0.04053333401679993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,1536,8,0,0.03656533360481262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,1536,32,0,0.03485333422819773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,1536,64,0,0.03454400102297465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,1536,16,0,0.0354720006386439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,1536,1,0,0.12707199652989706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,1536,2,0,0.0663679987192154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,1536,16,0,0.035317334036032356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,1536,32,0,0.035045333206653595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,1536,8,0,0.03621866554021835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,1536,64,0,0.03454400102297465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,1536,4,0,0.04227200150489807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,1536,2,0,0.06653866668542226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,1536,1,0,0.12661332885424295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,2048,8,0,0.04534933467706045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,2048,16,0,0.04277333120505015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,2048,64,0,0.04242133100827535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,2048,2,0,0.09969066580136617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,2048,4,0,0.055498664577802025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,2048,32,0,0.04254400233427683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,2048,1,0,0.19575466712315878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,2048,4,0,0.055386667450269066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,2048,8,0,0.045279999574025474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,2048,16,0,0.04318400224049886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,2048,32,0,0.04257066547870636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,2048,64,0,0.04200533529122671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,2048,1,0,0.19316800435384116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,2048,2,0,0.09918399651845296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,3072,16,0,0.05895466605822245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,3072,4,0,0.10282133022944133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,3072,64,0,0.05764266848564148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,3072,8,0,0.06387199958165486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,3072,32,0,0.05821333328882853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,3072,2,0,0.20563199122746786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,3072,8,0,0.06408533453941345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,3072,16,0,0.05916266640027364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,3072,32,0,0.05801600217819214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,3072,2,0,0.20625066757202148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,3072,1,0,0.36536534627278644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,3072,4,0,0.10693333546320598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,3072,64,0,0.057461331288019814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,3072,1,0,0.3722240130106608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,4096,8,0,0.08566932876904805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,4096,16,0,0.07533333202203114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,4096,32,0,0.07337599992752075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,4096,64,0,0.0726453314224879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,4096,4,0,0.15743999679883322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,4096,2,0,0.32414400577545166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,4096,16,0,0.0754559983809789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,4096,8,0,0.08706133564313252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,4096,1,0,0.600272019704183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,4096,4,0,0.1558613379796346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,4096,32,0,0.07337599992752075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,4096,2,0,0.32473599910736084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,4096,64,0,0.07275199890136719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,4096,1,0,0.6002773443857828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,6144,8,0,0.17640533049901327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,6144,16,0,0.11201066772143047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,6144,4,0,0.35813331604003906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,6144,32,0,0.10497066378593445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,6144,64,0,0.1035146713256836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,6144,2,0,0.655626654624939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,6144,8,0,0.1770026683807373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,6144,32,0,0.1060693363348643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,6144,16,0,0.11241066455841064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,6144,4,0,0.35674134890238446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,6144,1,0,1.2606879870096843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,6144,64,0,0.1030560036500295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,6144,2,0,0.6476426521937052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,8192,8,0,0.28143467505772907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,8192,16,0,0.14851199587186178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,8192,4,0,0.5882826646169027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,8192,32,0,0.13991467157999674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,6144,1,0,1.2306719621022542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,8192,64,0,0.13403200109799704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,8192,2,0,1.0937919616699219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,8192,32,0,0.13989333311716715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,8192,16,0,0.15052266915639242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,8192,4,0,0.5888266563415527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,8192,8,0,0.2762826681137085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,8192,64,0,0.13391466935475668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,8192,1,0,2.109546661376953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,8192,2,0,1.083456039428711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,10240,32,0,0.1743519902229309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,10240,16,0,0.23647467295328775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,10240,8,0,0.4946133295694987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,10240,4,0,0.8717919985453287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,10240,64,0,0.16752000649770102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,8192,1,0,2.109658718109131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,10240,2,0,1.6439733505249023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,10240,4,0,0.8718506495157877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,10240,16,0,0.23458667596181235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,10240,8,0,0.49428268273671466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,10240,32,0,0.17376534144083658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,10240,64,0,0.16827734311421713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,10240,2,0,1.6456586519877117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,10240,1,0,3.28873602549235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,12288,8,0,0.672048012415568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,12288,4,0,1.223855972290039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,12288,32,0,0.2116746703783671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,12288,16,0,0.3249066670735677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,12288,64,0,0.20174932479858398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,10240,1,0,3.2462027867635093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,12288,2,0,2.3068906466166177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,12288,16,0,0.32817065715789795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,12288,4,0,1.2244160175323486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,12288,8,0,0.6733173529307047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,12288,32,0,0.20889067649841309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,12288,64,0,0.20412800709406534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,12288,2,0,2.359941323598226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,12288,1,0,4.728975931803386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,16384,8,0,1.1269653638203938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,16384,16,0,0.5345653295516968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,16384,4,0,2.0559520721435547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,16384,64,0,0.2704746723175049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,16384,32,0,0.2760213414827983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,12288,1,0,4.642143885294597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,16384,2,0,4.028709411621094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,16384,16,0,0.5232160091400146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,16384,8,0,1.1295519669850667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,16384,32,0,0.2741439938545227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,16,1,0,0.015008000036080679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,16384,64,0,0.2727839946746826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,16384,4,0,2.0865440368652344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,16,8,0,0.01301866645614306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,16,2,0,0.013893333574136099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,16,4,0,0.013269333789745966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,16,16,0,0.012885333349307379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,16,32,0,0.012458667159080505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,16,64,0,0.01268799975514412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,16,1,0,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,16,2,0,0.013989333063364029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,16,4,0,0.013301332791646322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,16384,2,0,4.035706520080566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,16,8,0,0.01314666618903478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,16,16,0,0.012608000387748083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,16,32,0,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,16,64,0,0.012517333030700684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,32,1,0,0.015024000157912573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,16384,1,0,8.273904164632162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,32,2,0,0.013967999567588171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,32,4,0,0.013455999394257864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,32,8,0,0.013466666142145792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,32,16,0,0.012863999853531519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,32,32,0,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,32,64,0,0.013088000317414602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,32,1,0,0.014661333213249842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,32,2,0,0.013994666437307993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,32,8,0,0.012847999731699625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,32,4,0,0.014767999450365702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,32,16,0,0.012746666868527731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,32,32,0,0.012863999853531519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,32,64,0,0.012863999853531519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,64,2,0,0.014767999450365702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,64,1,0,0.016458666572968166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,64,4,0,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,64,8,0,0.014165333161751429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,64,16,0,0.014165333161751429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,16384,1,0,8.702192306518555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,64,64,0,0.013514666507641474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,64,32,0,0.013647999614477158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,64,1,0,0.01653333380818367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,64,2,0,0.014826666563749313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,64,4,0,0.014533333480358124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,64,8,0,0.01754666616519292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,64,16,0,0.013989333063364029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,64,64,0,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,64,32,0,0.01392000044385592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,128,1,0,0.019306667149066925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,128,2,0,0.0170666662355264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,128,4,0,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,128,8,0,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,128,16,0,0.015205333630243937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,128,32,0,0.014789332946141561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,128,64,0,0.014831999937693277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,128,1,0,0.019173332800467808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,128,2,0,0.017082666357358296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,128,8,0,0.015791999797026317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,128,32,0,0.014778666198253632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,128,4,0,0.015706667055686314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,128,16,0,0.015216000378131866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,128,64,0,0.014826666563749313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,256,1,0,0.03145066648721695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,256,2,0,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,256,4,0,0.017877332866191864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,256,8,0,0.017103999853134155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,256,16,0,0.01674666628241539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,256,64,0,0.016149333367745083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,256,32,0,0.01623999948302905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,256,32,0,0.016154666741689045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,256,16,0,0.01724799970785777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,256,1,0,0.031386665999889374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,256,64,0,0.016186666985352833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,256,4,0,0.01807466646035512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,256,2,0,0.019946667055288952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,256,8,0,0.017173333714405697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,512,4,0,0.02327466756105423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,512,2,0,0.03537066777547201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,512,1,0,0.0695306658744812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,512,16,0,0.021381333470344543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,512,32,0,0.020138667275508244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,512,64,0,0.019440000255902607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,512,8,0,0.021253332495689392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,512,4,0,0.02275199939807256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,512,8,0,0.021317332983016968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,512,1,0,0.06899733344713847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,512,16,0,0.0210506667693456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,512,32,0,0.020538666596015293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,512,64,0,0.019786667078733444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,512,2,0,0.034815999368826546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,1024,8,0,0.03046400099992752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,1024,32,0,0.028016000986099243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,1024,4,0,0.042117332418759666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,1024,16,0,0.029002666473388672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,1024,64,0,0.027717334528764088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,1024,2,0,0.07749333480993907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,1024,1,0,0.13004266222318014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,1024,64,0,0.02789866675933202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,1024,8,0,0.030479999879995983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,1024,16,0,0.02870933214823405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,1024,32,0,0.02828799933195114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,1024,4,0,0.04190400242805481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,1024,2,0,0.07814933359622955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,1024,1,0,0.8354559739430746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,1536,16,0,0.037061333656311035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,1536,64,0,0.035760000348091125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,1536,8,0,0.04124800115823746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,1536,32,0,0.03629866739114126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,1536,4,0,0.0680213322242101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,1536,2,0,0.12963733077049255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,1536,1,0,0.2310453255971273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,1536,8,0,0.04223466912905375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,1536,16,0,0.03732266773780187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,1536,32,0,0.03598399957021078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,1536,4,0,0.0680159976085027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,1536,64,0,0.03527999917666117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,1536,2,0,0.13057066996892294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,1536,1,0,0.22892266511917114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,2048,16,0,0.04698666433493296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,2048,8,0,0.05700799822807312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,2048,32,0,0.04437866806983948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,2048,4,0,0.10231467088063557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,2048,2,0,0.19631999731063843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,2048,64,0,0.04340266684691111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,2048,8,0,0.057946667075157166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,2048,4,0,0.10313600301742554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,2048,32,0,0.04385599990685781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,2048,1,0,0.36294400691986084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,2048,16,0,0.047728002071380615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,2048,2,0,0.1962560017903646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,2048,64,0,0.04338666796684265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,2048,1,0,0.35808531443277997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,3072,16,0,0.0674720009167989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,3072,32,0,0.06169066826502482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,3072,8,0,0.10648533701896667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,3072,4,0,0.212826669216156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,3072,64,0,0.059445331494013466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,3072,2,0,0.37176533540089923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,3072,16,0,0.06816533207893372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,3072,8,0,0.10897066195805867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,3072,32,0,0.06112533311049143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,3072,64,0,0.059301331639289856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,3072,4,0,0.20996799071629843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,3072,2,0,0.37002134323120117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,3072,1,0,0.7055413722991943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,4096,32,0,0.08155733346939087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,4096,16,0,0.09009066224098206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,4096,8,0,0.16293332974116007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,4096,4,0,0.3349866469701131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,3072,1,0,0.7024160226186117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,4096,64,0,0.07580799857775371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,4096,2,0,0.606106678644816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,4096,32,0,0.08044266700744629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,4096,8,0,0.16385066509246826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,4096,16,0,0.09070400396982829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,4096,4,0,0.33059199651082355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,4096,64,0,0.07598933577537537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,4096,2,0,0.6018240054448446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,4096,1,0,1.1893760363260906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,6144,32,0,0.12043733398119609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,6144,8,0,0.37252267201741535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,6144,16,0,0.1837386687596639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,6144,4,0,0.6612693468729655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,4096,1,0,1.154314676920573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,6144,64,0,0.11361599961916606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,6144,2,0,1.2487200101216633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,6144,4,0,0.6611040035883585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,6144,16,0,0.18657066424687704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,6144,8,0,0.37198400497436523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,6144,32,0,0.11949333548545837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,6144,64,0,0.1132586697737376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,6144,2,0,1.232357343037923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,6144,1,0,2.4205172856648765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,8192,8,0,0.6024213234583536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,8192,32,0,0.15838932991027832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,8192,4,0,1.1052052974700928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,8192,16,0,0.28562132517496747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,8192,64,0,0.1500640014807383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,8192,2,0,2.0947200457255044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,8192,8,0,0.6037973165512085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,6144,1,0,2.43233060836792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,8192,32,0,0.15802133083343506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,8192,64,0,0.1492693324883779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,8192,4,0,1.09388264020284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,8192,16,0,0.285098671913147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,8192,2,0,2.1126933097839355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,10240,8,0,0.8852372964223226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,10240,32,0,0.241866668065389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,8192,1,0,4.367472012837728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,10240,4,0,1.6433653831481934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,10240,16,0,0.5088533163070679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,10240,64,0,0.1844373345375061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,8192,1,0,4.41103998819987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,10240,8,0,0.8872266610463461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,10240,2,0,3.214720090230306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,10240,4,0,1.6639520327250164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,10240,64,0,0.1852160096168518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,10240,32,0,0.24154132604599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,10240,16,0,0.5053280194600424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,10240,2,0,3.182565371195475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,12288,8,0,1.2382666269938152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,12288,16,0,0.6867413520812988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,10240,1,0,6.678927739461263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,12288,32,0,0.3380746841430664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,12288,4,0,2.313642660776774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,12288,64,0,0.21974400679270426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,12288,2,0,4.5850880940755205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,12288,8,0,1.2250453631083171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,10240,1,0,6.588725407918294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,12288,4,0,2.3134454091389975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,12288,32,0,0.3360639810562134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,12288,64,0,0.2237493395805359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,12288,16,0,0.6916639804840088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,12288,2,0,4.778266588846843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,16384,8,0,2.1029225985209146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,16384,16,0,1.1393120288848877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,16384,32,0,0.539354681968689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,16384,4,0,4.332874615987142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,16384,64,0,0.2884640097618103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,12288,1,0,9.542224248250326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,16384,2,0,8.310869216918945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,12288,1,0,9.703893025716146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,16384,8,0,2.1137439409891763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,16384,4,0,4.219663937886556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,16384,32,0,0.5358826716740926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,16384,16,0,1.1309973398844402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,16384,64,0,0.2913653254508972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,16,1,0,0.018378666291634243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,16,2,0,0.014549333602190018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,16,4,0,0.013845333208640417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,16,8,0,0.013754667093356451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,16,16,0,0.013088000317414602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,16,32,0,0.012842666357755661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,16,64,0,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,16,1,0,0.017925333231687546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,16,2,0,0.01443733274936676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,16,4,0,0.013717333475748697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,16,8,0,0.013354666531085968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,16,16,0,0.01322666679819425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,16,32,0,0.012666666259368261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,16,64,0,0.013077333569526672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,32,1,0,0.018464000274737675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,32,2,0,0.014815999815861383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,16384,2,0,8.47823969523112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,32,4,0,0.014298666268587112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,32,8,0,0.013605333864688873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,32,16,0,0.013088000317414602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,32,32,0,0.013141332815090815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,32,64,0,0.013173333058754602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,32,1,0,0.01842133328318596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,32,2,0,0.014789332946141561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,32,8,0,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,32,4,0,0.014005333185195923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,32,16,0,0.013072000195582708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,32,32,0,0.012928000340859095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,32,64,0,0.01301866645614306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,64,1,0,0.02093333254257838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,64,2,0,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,64,4,0,0.014954666296641031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,64,16,0,0.014186666657527288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,64,8,0,0.014666666587193808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,64,32,0,0.01431999976436297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,64,64,0,0.013594667116800943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,16384,1,0,16.763472239176433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,64,2,0,0.016682667036851246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,64,1,0,0.07000533243020375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,64,4,0,0.014954666296641031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,64,8,0,0.014831999937693277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,64,16,0,0.014138666292031607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,64,32,0,0.01404800017674764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,64,64,0,0.013888000200192133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,128,4,0,0.01691199963291486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,128,2,0,0.019850666324297588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,128,1,0,0.03179733455181122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,128,8,0,0.016069332758585613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,128,16,0,0.015717333803574245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,128,32,0,0.01532799998919169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,128,64,0,0.01532799998919169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,128,2,0,0.019845332950353622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,128,4,0,0.017136000096797943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,128,1,0,0.03173866619666418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,128,8,0,0.016176000237464905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,128,16,0,0.015557333827018738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,128,32,0,0.01533866673707962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,128,64,0,0.015322666615247726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,256,16,0,0.017717332889636356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,256,4,0,0.019968000551064808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,256,8,0,0.018485333770513535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,256,1,0,0.06051200131575266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,256,32,0,0.01661866654952367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,256,2,0,0.033071999748547874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,256,64,0,0.01632000009218852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,16384,1,0,16.867652893066406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,256,2,0,0.03367999941110611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,256,16,0,0.01721599946419398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,256,4,0,0.03499199946721395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,256,32,0,0.016890666137139004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,256,1,0,0.06061866879463196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,256,8,0,0.0179626668492953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,256,64,0,0.01659199967980385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,512,64,0,0.0204373337328434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,512,16,0,0.021541332205136616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,512,32,0,0.020970667401949566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,512,8,0,0.023962666591008503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,512,4,0,0.035546667873859406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,512,2,0,0.07136533161004384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,512,1,0,0.10133333007494609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,512,8,0,0.023813332120577495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,512,64,0,0.020549333343903225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,512,16,0,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,512,32,0,0.021375998854637146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,512,4,0,0.03610666592915853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,512,2,0,0.07088533540566762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,512,1,0,0.10083733002344768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,1024,64,0,0.02889599899450938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,1024,32,0,0.02958933264017105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,1024,8,0,0.04426133135954539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,1024,16,0,0.03225066761175791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,1024,4,0,0.07996800045172374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,1024,2,0,0.13245866696039835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,1024,16,0,0.032127998769283295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,1024,32,0,0.029781334102153778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,1024,8,0,0.044405331214269005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,1024,4,0,0.0802346666653951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,1024,1,0,0.239247997601827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,1024,2,0,0.13121066490809122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,1024,64,0,0.028783999383449554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,1536,16,0,0.04530133306980133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,1024,1,0,0.23989333709081015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,1536,8,0,0.07081066568692525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,1536,32,0,0.03958400090535482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,1536,4,0,0.13295466701189676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,1536,2,0,0.23682665824890137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,1536,64,0,0.03728533287843069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,1536,8,0,0.07083199918270111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,1536,16,0,0.04524800181388855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,1536,32,0,0.0391146664818128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,1536,1,0,0.43327999114990234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,1536,4,0,0.13168000181516012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,1536,64,0,0.03755733370780945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,1536,2,0,0.23329599698384604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,2048,8,0,0.10693333546320598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,2048,32,0,0.05162133276462555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,2048,4,0,0.20269334316253662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,1536,1,0,0.43451201915740967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,2048,16,0,0.06065066655476888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,2048,64,0,0.046069333950678505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,2048,2,0,0.36057066917419434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,2048,8,0,0.10690666238466899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,2048,16,0,0.060922667384147644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,2048,32,0,0.05153599878152212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,2048,4,0,0.20188800493876138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,2048,64,0,0.04640533526738485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,2048,2,0,0.36374934514363605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,2048,1,0,0.7005706628163656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,3072,16,0,0.11339733004570007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,3072,8,0,0.21743466456731161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,3072,4,0,0.3796159823735555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,2048,1,0,0.6901653607686361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,3072,64,0,0.06897599995136261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,3072,32,0,0.07282133400440216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,3072,16,0,0.11404266953468323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,3072,2,0,0.705024003982544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,3072,8,0,0.21628799041112265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,3072,4,0,0.37796799341837567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,3072,64,0,0.06891733407974243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,3072,32,0,0.07256533205509186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,3072,2,0,0.7116693655649821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,4096,8,0,0.34239999453226727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,3072,1,0,1.362602710723877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,4096,16,0,0.17214399576187134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,4096,4,0,0.6113226811091105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,4096,32,0,0.09769066174825032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,4096,64,0,0.0883146623770396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,3072,1,0,1.374010721842448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,4096,2,0,1.1710453033447266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,4096,16,0,0.1717546582221985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,4096,8,0,0.33606934547424316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,4096,64,0,0.08877866466840108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,4096,32,0,0.0969493289788564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,4096,4,0,0.6235626538594564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,4096,2,0,1.1621387004852295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,4096,1,0,2.3000906308492026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,6144,8,0,0.6796320279439291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,6144,16,0,0.38122133413950604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,6144,32,0,0.19472533464431763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,6144,64,0,0.1308746635913849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,6144,4,0,1.2597653071085613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,4096,1,0,2.307194709777832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,6144,8,0,0.6714133421579996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,6144,16,0,0.3794026772181193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,6144,2,0,2.440762678782145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,6144,32,0,0.19581333796183267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,6144,4,0,1.2780906359354656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,6144,64,0,0.13025599718093872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,6144,2,0,2.427365303039551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,8192,8,0,1.119658629099528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,8192,32,0,0.30052266518274945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,8192,16,0,0.6128640174865723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,8192,4,0,2.161909262339274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,8192,64,0,0.1714293360710144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,6144,1,0,4.900224049886067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,8192,2,0,4.1514027913411455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,8192,8,0,1.1079253355662029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,6144,1,0,4.980256080627441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,8192,64,0,0.17291200160980225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,8192,32,0,0.2994719942410787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,8192,16,0,0.614741325378418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,8192,4,0,2.1337013244628906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,16,1,0,0.024842667082945507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,16,2,0,0.017722666263580322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,16,4,0,0.014954666296641031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,16,8,0,0.01402666668097178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,16,16,0,0.013343999783198038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,16,32,0,0.013045333325862885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,16,64,0,0.012794667234023413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,16,1,0,0.02447466552257538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,16,2,0,0.017456000049908955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,16,4,0,0.014762666076421738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,16,8,0,0.013957332819700241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,16,16,0,0.013424000392357508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,16,32,0,0.013141332815090815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,32,1,0,0.025061334172884624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,16,64,0,0.012527999778588613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,8192,2,0,4.351215998331706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,32,2,0,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,32,4,0,0.01479999969402949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,32,8,0,0.01404800017674764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,32,16,0,0.013493333011865616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,32,32,0,0.013503999759753546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,32,64,0,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,32,1,0,0.024853333830833435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,32,2,0,0.018250666558742523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,32,4,0,0.01471466695268949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,32,8,0,0.013823999712864557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,32,16,0,0.013850666582584381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,32,32,0,0.013274667163689932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,32,64,0,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,8192,1,0,8.790719985961914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,64,4,0,0.01655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,64,1,0,0.03241066634654999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,64,2,0,0.02077866718173027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,64,8,0,0.01479999969402949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,64,16,0,0.014709333578745524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,64,32,0,0.01444799949725469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,64,64,0,0.01402666668097178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,64,2,0,0.020714666694402695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,64,1,0,0.03278933217128118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,64,4,0,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,64,8,0,0.015061333775520325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,64,16,0,0.014837333311637243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,64,32,0,0.014309333016475042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,64,64,0,0.013957332819700241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,128,4,0,0.019914666811625164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,128,1,0,0.05622933308283488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,128,8,0,0.016869333883126576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,128,2,0,0.03310399999221166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,128,16,0,0.016501333564519882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,128,32,0,0.015664000064134598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,128,64,0,0.028223998844623566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,128,8,0,0.016906666258970898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,128,32,0,0.017407999684413273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,128,16,0,0.016016000260909397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,128,4,0,0.019776000330845516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,128,1,0,0.056074668963750206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,128,2,0,0.033071999748547874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,128,64,0,0.015279999623696009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,8192,1,0,8.706917444864908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,256,8,0,0.02042666698495547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,256,4,0,0.03357866654793421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,256,16,0,0.01858666663368543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,256,32,0,0.01823466643691063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,256,2,0,0.06296533346176147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,256,64,0,0.016693333784739178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,256,1,0,0.09453866879145305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,256,16,0,0.018954666952292126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,256,8,0,0.036464000741640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,256,4,0,0.033743999898433685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,256,2,0,0.061946665247281395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,256,32,0,0.017317333569129307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,256,64,0,0.0174346665541331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,256,1,0,0.09474666913350423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,512,32,0,0.022639999787012737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,512,64,0,0.021562665700912476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,512,16,0,0.024890666206677754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,512,8,0,0.03822933385769526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,512,4,0,0.07422400017579396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,512,2,0,0.1032319962978363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,512,8,0,0.03772266705830892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,512,4,0,0.07313600182533264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,512,32,0,0.022202665607134502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,512,1,0,0.18524799744288126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,512,16,0,0.024698667228221893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,512,2,0,0.10156266887982686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,512,64,0,0.02186133215824763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,512,1,0,0.18445332845052084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,1024,32,0,0.03696000079313914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,1024,16,0,0.04826133449872335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,1024,4,0,0.13800533612569174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,1024,8,0,0.08451199531555176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,1024,64,0,0.03161599983771642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,1024,2,0,0.24224533637364706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,1024,16,0,0.048570667703946434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,1024,8,0,0.08496532837549846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,1024,4,0,0.13734933733940125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,1024,32,0,0.036831999818483986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,1024,64,0,0.03179733455181122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,1024,1,0,0.45374401410420734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,1024,2,0,0.24102399746576944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,1536,8,0,0.13923733433087668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,1536,32,0,0.050399998823801674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,1536,16,0,0.07727466523647308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,1024,1,0,0.45123199621836346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,1536,4,0,0.23877867062886557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,1536,64,0,0.046949331959088646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,1536,16,0,0.0775626649459203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,1536,2,0,0.444538672765096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,1536,8,0,0.13847999771436056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,1536,32,0,0.05083199838797251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,1536,64,0,0.04667733112970988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,1536,4,0,0.2372693419456482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,1536,2,0,0.43807466824849445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,2048,16,0,0.11408533652623494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,1536,1,0,0.8476640383402506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,2048,8,0,0.2072746753692627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,2048,4,0,0.37380798657735187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,2048,32,0,0.06834133466084798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,2048,64,0,0.059392000238100685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,2048,2,0,0.7005440394083658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,1536,1,0,0.8406026363372803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,2048,8,0,0.20805867513020834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,2048,4,0,0.36826666196187335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,2048,32,0,0.06871466835339864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,2048,16,0,0.11498133341471355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,2048,64,0,0.059301331639289856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,2048,2,0,0.7095306714375814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,3072,8,0,0.38787198066711426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,3072,16,0,0.22938666741053262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,2048,1,0,1.3614187240600586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,3072,4,0,0.7228480180104574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,3072,32,0,0.12386666735013326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,3072,64,0,0.08331733445326488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,2048,1,0,1.3466240564982097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,3072,8,0,0.3940853277842204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,3072,4,0,0.7193973064422607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,3072,32,0,0.12458667159080505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,3072,16,0,0.2256106734275818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,3072,64,0,0.08384000261624654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,3072,2,0,1.3719360033671062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,3072,2,0,1.3861974080403645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,3072,1,0,2.737039883931478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,4096,16,0,0.3509226640065511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,4096,8,0,0.6235146522521973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,4096,32,0,0.18710933128992716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,4096,4,0,1.1833173433939617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,4096,64,0,0.11101333300272624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,4096,8,0,0.6292426586151123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,3072,1,0,2.734287897745768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,4096,2,0,2.2817066510518393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,4096,32,0,0.1869493325551351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,4096,16,0,0.35089067618052167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,4096,4,0,1.1839306354522705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,4096,64,0,0.11091199517250061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,16,1,0,0.03619199991226196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,16,2,0,0.02399466683467229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,16,4,0,0.01777600000301997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,16,8,0,0.014677333335081736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,16,16,0,0.013637332866589228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,16,32,0,0.013471999516089758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,16,64,0,0.01333333303531011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,16,2,0,0.023706667125225067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,16,1,0,0.0356480007370313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,16,4,0,0.018330667167901993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,16,8,0,0.014991999914248785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,16,16,0,0.013712000101804733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,16,32,0,0.01340266689658165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,16,64,0,0.01331199953953425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,32,1,0,0.040864000717798867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,4096,2,0,2.2965866724650064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,32,4,0,0.018346666047970455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,32,8,0,0.015125333021084467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,32,2,0,0.024821333587169647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,32,16,0,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,32,32,0,0.01370666672786077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,32,64,0,0.01339200014869372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,32,2,0,0.025008000433444977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,32,1,0,0.04125866790612539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,32,8,0,0.014943999548753103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,32,4,0,0.01836266616980235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,32,16,0,0.013962666193644205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,32,64,0,0.013349333157142004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,32,32,0,0.013658666362365087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,64,2,0,0.033520000676314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,64,1,0,0.05486933390299479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,64,8,0,0.028815999627113342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,64,4,0,0.020992000897725422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,64,16,0,0.015077333897352219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,64,64,0,0.014069333672523499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,64,32,0,0.014890667051076889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,64,4,0,0.021055998901526134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,64,2,0,0.03368533402681351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,64,1,0,0.055104002356529236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,64,16,0,0.015274666249752045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,64,8,0,0.016751999656359356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,4096,1,0,4.779413223266602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,64,32,0,0.014746667196353277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,64,64,0,0.014495999862750372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,128,4,0,0.03383466601371765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,128,8,0,0.02093333254257838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,128,2,0,0.05774933099746704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,128,16,0,0.017456000049908955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,128,32,0,0.01700266698996226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,128,1,0,0.10026133060455322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,128,64,0,0.015770666301250458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,128,16,0,0.017786666750907898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,128,8,0,0.02067199970285098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,128,32,0,0.016522667060295742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,128,4,0,0.03364799916744232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,128,2,0,0.058464000622431435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,128,1,0,0.09915199875831604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,128,64,0,0.01658133293191592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,256,4,0,0.0643146683772405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,256,32,0,0.019226666539907455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,256,16,0,0.021909333765506744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,256,8,0,0.03678400069475174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,256,2,0,0.09728533029556274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,256,64,0,0.018618666877349217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,256,8,0,0.037231999138991036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,256,1,0,0.17188799381256104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,256,4,0,0.06437333424886067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,256,16,0,0.0220266655087471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,256,2,0,0.09732266267140706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,256,32,0,0.0194560003777345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,256,64,0,0.01929066702723503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,256,1,0,0.17151999473571777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,512,8,0,0.07864533364772797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,512,16,0,0.042208001017570496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,512,32,0,0.029663999875386555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,4096,1,0,4.502554575602214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,512,4,0,0.10776000221570332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,512,64,0,0.024442667762438457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,512,2,0,0.18648000558217367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,512,16,0,0.04318400224049886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,512,8,0,0.07834133505821228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,512,32,0,0.03001066545645396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,512,4,0,0.10590400298436482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,512,64,0,0.024853333830833435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,512,2,0,0.18755199511845908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,512,1,0,0.3450293143590291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,1024,32,0,0.056314667065938316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,1024,16,0,0.09204799930254619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,1024,8,0,0.14386133352915445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,512,1,0,0.3455093304316203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,1024,64,0,0.04472533365090688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,1024,4,0,0.247216006120046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,1024,8,0,0.14285332957903543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,1024,2,0,0.46107733249664307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,1024,16,0,0.10027199983596802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,1024,4,0,0.24753065903981528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,1024,64,0,0.045194665590922035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,1024,32,0,0.05650666852792104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,1024,2,0,0.4597333272298177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,1536,8,0,0.2474506696065267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,1024,1,0,0.9041066964467367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,1536,16,0,0.1500640014807383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,1536,64,0,0.0611413319905599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,1536,32,0,0.08869333068529765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,1536,4,0,0.4505759874979655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,1024,1,0,0.8914133707682291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,1536,16,0,0.1483626663684845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,1536,8,0,0.2500320076942444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,1536,2,0,0.8528160254160563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,1536,32,0,0.08814932902654012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,1536,4,0,0.4495999813079834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,1536,64,0,0.061050668358802795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,1536,2,0,0.871338685353597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,2048,8,0,0.38392531871795654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,2048,16,0,0.22200000286102295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,2048,32,0,0.1281013290087382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,2048,4,0,0.709274689356486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,2048,64,0,0.08171733220418294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,1536,1,0,1.661146640777588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,1536,1,0,1.6792213122049968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,2048,16,0,0.2210879921913147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,2048,2,0,1.3752800623575847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,2048,4,0,0.7219680150349935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,2048,8,0,0.3780906597773234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,2048,64,0,0.08191999793052673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,2048,32,0,0.12813333670298258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,16,1,0,0.06169066826502482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,16,4,0,0.024005333582560223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,16,2,0,0.0354666660229365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,16,8,0,0.01791999985774358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,16,16,0,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,16,32,0,0.01393066719174385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,16,1,0,0.06070933242638906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,16,64,0,0.013541333377361298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,16,2,0,0.035589332381884255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,16,4,0,0.024005333582560223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,2048,2,0,1.371338685353597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,16,8,0,0.017765333255132038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,16,64,0,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,16,32,0,0.013797332843144735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,16,16,0,0.014736000448465347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,32,2,0,0.04173333446184794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,32,4,0,0.025306666890780132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,32,1,0,0.06965866684913635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,32,8,0,0.01810666670401891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,32,16,0,0.01524266724785169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,32,32,0,0.014282666146755219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,32,64,0,0.014192000031471252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,32,2,0,0.041402667760849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,32,1,0,0.06914133330186208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,32,8,0,0.01871466636657715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,32,16,0,0.01525866612792015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,32,4,0,0.02462399999300639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,32,32,0,0.014058666924635569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,32,64,0,0.013845333208640417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,2048,1,0,2.7340854008992515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,64,2,0,0.05602666735649109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,64,8,0,0.021514666577180225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,64,16,0,0.01754133279124896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,64,1,0,0.09818666179974873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,64,4,0,0.03450666616360346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,64,32,0,0.015749332805474598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,64,64,0,0.015450666348139444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,64,16,0,0.01759999990463257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,64,8,0,0.021717332303524017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,64,4,0,0.03461333364248276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,64,32,0,0.01594666639963786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,64,2,0,0.056074668963750206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,64,1,0,0.09807466467221577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,64,64,0,0.015594666202863058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,128,16,0,0.02176533391078313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,128,8,0,0.03587199995915095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,128,4,0,0.06017066538333893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,128,32,0,0.018288000176350277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,128,2,0,0.10420266787211101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,2048,1,0,2.728384017944336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,128,64,0,0.017551999539136887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,128,1,0,0.18503467241923013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,128,8,0,0.03563733398914337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,128,4,0,0.060645331939061485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,128,32,0,0.018058666338523228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,128,16,0,0.021695998807748158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,128,2,0,0.10187733173370361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,128,64,0,0.02056533346573512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,128,1,0,0.1844373345375061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,256,16,0,0.041850666205088295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,256,8,0,0.06886399785677592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,256,32,0,0.02609066665172577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,256,4,0,0.10059733192125957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,256,64,0,0.021546666820844013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,256,2,0,0.17541333039601645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,256,16,0,0.04161066561937332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,256,8,0,0.06901866694291432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,256,4,0,0.10040000081062317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,256,64,0,0.022122666239738464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,256,32,0,0.02644266684850057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,256,2,0,0.17554134130477905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,256,1,0,0.32474666833877563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,512,16,0,0.08733333150545756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,512,8,0,0.11361066500345866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,512,32,0,0.051701332132021584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,256,1,0,0.32282666365305585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,512,64,0,0.03925866633653641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,512,4,0,0.19340799252192178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,512,16,0,0.08760000268618266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,512,8,0,0.11348266402880351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,512,2,0,0.35119465986887616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,512,4,0,0.19434134165445963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,512,32,0,0.05154666801293691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,512,64,0,0.03925333420435587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,512,2,0,0.35123733679453534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,512,1,0,0.6798453330993652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,1024,8,0,0.26394132773081463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,1024,16,0,0.15661866466204324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,1024,32,0,0.10599467158317566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,1024,64,0,0.06974400083223979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,1024,4,0,0.4728960196177165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,512,1,0,0.66921067237854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,1024,16,0,0.15658133228619894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,1024,8,0,0.2609226703643799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,1024,2,0,0.9073279698689779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,1024,32,0,0.10563199718793233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,1024,64,0,0.06983466446399689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,16,1,0,0.10991999506950378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,1024,4,0,0.4747840166091919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,16,2,0,0.06252266466617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,16,4,0,0.03549333413441976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,16,8,0,0.023743999501069386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,16,16,0,0.017792000124851864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,16,32,0,0.014720000326633453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,64,16,64,0,0.014015999933083853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,16,1,0,0.11079999804496765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,16,2,0,0.06225599845250448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,16,4,0,0.03565866748491923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,16,8,0,0.02386666586001714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,16,16,0,0.018005333840847015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,16,32,0,0.014762666076421738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,64,16,64,0,0.013850666582584381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,1024,2,0,0.9189813137054443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,32,4,0,0.042591998974482216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,32,8,0,0.02603200078010559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,32,2,0,0.06961066524187724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,32,1,0,0.13114133477210999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,32,16,0,0.01871466636657715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,64,32,64,0,0.014682666709025701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,32,32,0,0.019653332730134327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,32,2,0,0.06895466645558675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,32,4,0,0.04221866528193156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,32,8,0,0.025583999852339428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,32,1,0,0.1325759987036387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,32,16,0,0.018778666853904724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,64,32,64,0,0.014581333845853806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,32,32,0,0.015626666446526844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,64,16,0,0.023018665611743927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,64,8,0,0.037061333656311035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,64,4,0,0.05735466877619425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,64,2,0,0.10307199756304423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,64,32,0,0.01823466643691063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,64,64,64,0,0.01621866722901662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,64,1,0,0.18147200345993042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,64,8,0,0.03644266724586487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,64,4,0,0.05793066819508871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,64,2,0,0.10262399911880493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,64,16,0,0.02499733368555705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,64,32,0,0.018378666291634243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,64,1,0,0.18289599816004434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,64,64,64,0,0.016384000579516094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,128,16,0,0.040261333187421165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,128,8,0,0.06486399968465169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,1024,1,0,1.7888906796773274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,128,4,0,0.10619200269381206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,128,32,0,0.026367999613285065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,128,2,0,0.19107200702031454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,64,128,64,0,0.02075733368595441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,128,8,0,0.0641546646753947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,128,16,0,0.04027199993530909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,128,1,0,0.3505226771036784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,128,4,0,0.1074133316675822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,128,32,0,0.026874666412671406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,64,128,64,0,0.02057066683967908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,128,2,0,0.18951465686162314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,1024,1,0,1.7734026908874512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,256,32,0,0.05037866532802582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,256,16,0,0.07763200004895528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,256,8,0,0.10859733819961548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,128,1,0,0.35817599296569824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,64,256,64,0,0.04049599915742874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,256,4,0,0.1818986733754476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,256,16,0,0.07734933495521545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,256,2,0,0.33105067412058514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,256,8,0,0.10850666960080464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,256,4,0,0.1967946688334147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,64,256,64,0,0.03712533414363861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,256,32,0,0.050714666644732155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,256,2,0,0.3302453358968099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,512,8,0,0.20779732863108316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,512,16,0,0.12753066420555115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,256,1,0,0.6288373470306396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,512,32,0,0.1016533374786377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,64,512,64,0,0.06532266736030579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,256,1,0,0.6288373470306396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,512,4,0,0.36684266726175946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,512,16,0,0.1267359952131907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,512,8,0,0.20849066972732544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,512,2,0,0.6841493447621664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,512,32,0,0.10109333197275798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,64,512,64,0,0.06573333342870076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,512,4,0,0.36604265371958417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,16,1,0,0.2544693350791931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,16,2,0,0.11071466406186421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,16,4,0,0.06330133477846782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,16,8,0,0.03570133447647095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,16,16,0,0.02409599969784419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,16,32,0,0.01806933308641116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,128,16,64,0,0.015173333386580149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,16,8,0,0.036501333117485046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,16,2,0,0.11125333110491435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,16,4,0,0.06270933151245117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,16,1,0,0.2506293257077535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,16,32,0,0.01841066653529803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,16,16,0,0.024346667031447094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,128,16,64,0,0.015109332899252573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,32,8,0,0.04465066889921824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,512,2,0,0.6858826478322347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,32,4,0,0.07203199962774913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,32,16,0,0.02664000044266383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,32,2,0,0.13587199648221335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,128,32,64,0,0.01646399994691213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,32,32,0,0.019776000330845516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,32,1,0,0.2665226658185323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,32,8,0,0.04452799757321676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,32,4,0,0.07251733541488647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,32,16,0,0.026901334524154663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,32,2,0,0.13477333386739096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,32,32,0,0.019509332875410717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,128,32,64,0,0.016522667060295742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,32,1,0,0.26148800055185956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,64,8,0,0.06285333136717479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,64,16,0,0.04137066751718521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,64,4,0,0.10413333773612976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,64,32,0,0.02714666724205017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,64,2,0,0.1906399925549825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,128,64,64,0,0.020303999384244282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,64,8,0,0.06247999767462412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,64,1,0,0.34705066680908203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,64,4,0,0.10404266913731892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,512,1,0,1.354021390279134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,64,16,0,0.040847999354203544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,64,32,0,0.027818667391935985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,64,2,0,0.19225066900253296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,128,64,64,0,0.020618667205174763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,512,1,0,1.3586667378743489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,64,1,0,0.3489493529001872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,128,16,0,0.07316266496976216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,128,8,0,0.11462933818499248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,128,32,0,0.0537120004494985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,128,128,64,0,0.037130666275819145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,128,4,0,0.19549334049224854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,128,16,0,0.07231466472148895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,128,8,0,0.11488533020019531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,128,4,0,0.19583467642466226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,128,2,0,0.36473600069681805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,128,128,64,0,0.0369759996732076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,128,32,0,0.049925332268079124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,128,2,0,0.3621973196665446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,256,8,0,0.19568532705307007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,256,16,0,0.12318399548530579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,256,32,0,0.09153599540392558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,128,256,64,0,0.06545066833496094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,128,1,0,0.6871199607849121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,256,4,0,0.34354666868845624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,128,1,0,0.6998666922251383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,256,16,0,0.12293866276741028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,256,8,0,0.19455466667811075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,256,2,0,0.6428906520207723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,256,32,0,0.09239466985066731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,128,256,64,0,0.06564799944559734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,256,16,2,0,0.25549866755803424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,256,16,4,0,0.11390399932861328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,256,4,0,0.3464426596959432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,256,16,1,0,0.5051039854685465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,256,16,16,0,0.03743999948104223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,256,16,32,0,0.025487999121348064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,256,16,8,0,0.06529066463311513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,256,16,64,0,0.019440000255902607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,256,16,4,0,0.11294399698575337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,256,16,8,0,0.06481066842873891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,256,16,2,0,0.25251199801762897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,256,2,0,0.6429333289464315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,256,16,16,0,0.03839466720819473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,256,16,32,0,0.02535466601451238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,256,16,64,0,0.019066666563351948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,256,16,1,0,0.5062026580174764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,256,32,16,0,0.048938666780789696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,256,32,8,0,0.07668266693751018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,256,32,4,0,0.13884266217549643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,256,1,0,1.2835840384165447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,256,32,32,0,0.03120533376932144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,256,32,64,0,0.021957332889238994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,256,32,2,0,0.27082133293151855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,256,32,16,0,0.04850133260091146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,256,32,8,0,0.07781333227952321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,256,32,4,0,0.1360106666882833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,256,32,1,0,0.5343146721522013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,256,32,32,0,0.031685332457224526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,256,32,64,0,0.021888000269730885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,256,32,2,0,0.2681120038032532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,256,64,16,0,0.07162133355935414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,256,32,1,0,0.532970666885376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,256,64,8,0,0.11238400141398112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,256,64,64,0,0.03801066676775614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,256,64,32,0,0.05298133194446564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,256,1,0,1.2822293440500896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,256,64,4,0,0.19349332650502524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,256,64,16,0,0.07117866476376851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,256,64,2,0,0.36557332674662274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,256,64,8,0,0.12143466869990031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,256,64,32,0,0.04986133178075155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,256,64,4,0,0.19368533293406168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,256,64,64,0,0.03810133288304011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,256,64,2,0,0.36883731683095294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,256,64,1,0,0.6795146465301514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,256,128,8,0,0.21056000391642252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,256,128,16,0,0.12833600242932638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,256,128,32,0,0.08737599849700928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,256,128,64,0,0.06499733527501424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,256,128,4,0,0.3732373317082723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,256,64,1,0,0.6823946634928385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,256,128,16,0,0.12808533509572348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,256,128,8,0,0.20883200565973917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,256,128,2,0,0.726143995920817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,256,128,4,0,0.3707733154296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,256,128,32,0,0.08729599912961324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,256,128,64,0,0.0651039977868398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,16,1,0,0.014501333236694336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,16,2,0,0.013744000345468521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,16,16,0,0.012975999464591345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,16,8,0,0.013429333766301474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,16,4,0,0.01358933374285698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,16,32,0,0.012602667013804117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,16,64,0,0.012367999802033106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,16,1,0,0.014938666174809137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,16,2,0,0.013765333841244379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,16,4,0,0.01333333303531011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,16,8,0,0.0129120002190272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,16,16,0,0.01322666679819425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,16,32,0,0.012597333639860153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,16,64,0,0.012634667257467905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,32,1,0,0.014720000326633453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,32,2,0,0.023434666295846302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,32,4,0,0.013701333353916803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,32,8,0,0.01340266689658165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,32,16,0,0.012784000486135483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,32,32,0,0.012853333105643591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,256,128,2,0,0.71234663327535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,32,64,0,0.012762666990359625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,32,1,0,0.014853333433469137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,32,2,0,0.013957332819700241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,32,4,0,0.013818666338920593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,32,8,0,0.013552000125249227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,32,16,0,0.013306666165590286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,32,32,0,0.01259200026591619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,32,64,0,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,64,1,0,0.016645333419243496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,64,2,0,0.01471466695268949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,64,4,0,0.014831999937693277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,64,8,0,0.014111999422311783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,64,16,0,0.013514666507641474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,64,32,0,0.01381333296497663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,64,64,0,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,64,1,0,0.0164533331990242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,64,2,0,0.014783999572197596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,64,4,0,0.014202666779359182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,64,8,0,0.014463999619086584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,64,16,0,0.014117332796255747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,256,128,1,0,1.3837599754333496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,64,32,0,0.013616000612576803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,64,64,0,0.013429333766301474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,128,1,0,0.019381333142518997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,128,2,0,0.016805333395799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,128,4,0,0.015728000551462173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,128,16,0,0.015413332730531693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,128,8,0,0.01569066693385442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,128,32,0,0.01523200049996376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,128,64,0,0.014922666052977243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,128,1,0,0.01942933350801468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,128,2,0,0.01632533346613248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,128,4,0,0.015834666788578033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,128,8,0,0.04010133445262909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,128,16,0,0.015141333142916361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,128,32,0,0.01516266663869222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,128,64,0,0.015018666783968607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,256,1,0,0.03176533430814743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,256,2,0,0.019589333484570186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,256,4,0,0.017871999492247898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,256,16,0,0.01602666700879733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,256,8,0,0.018079999834299088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,256,32,0,0.016362667083740234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,256,64,0,0.016336000214020412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,256,2,0,0.019440000255902607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,256,8,0,0.016693333784739178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,256,16,0,0.016682667036851246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,256,1,0,0.03146133323510488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,256,4,0,0.017786666750907898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,256,32,0,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,256,128,1,0,1.3818292617797852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,256,64,0,0.01586666703224182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,512,4,0,0.022128000855445862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,512,8,0,0.021066665649414062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,512,32,0,0.020197333147128422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,512,16,0,0.020224000016848247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,512,64,0,0.02219199885924657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,512,2,0,0.034847999612490334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,512,1,0,0.06858133276303609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,512,32,0,0.020373333245515823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,512,2,0,0.034287999073664345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,512,8,0,0.020831999679406483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,512,64,0,0.01952533299724261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,512,16,0,0.020981334149837494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,512,4,0,0.02235200007756551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,512,1,0,0.06861333549022675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,1024,64,0,0.02754133443037669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,1024,16,0,0.028405333558718365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,1024,32,0,0.027984000742435455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,1024,8,0,0.029743999242782593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,1024,4,0,0.041877334316571556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,1024,2,0,0.07665599882602692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,1024,1,0,0.12873599926630655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,1024,8,0,0.029637334247430164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,1024,16,0,0.02826666583617528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,1024,32,0,0.02735999971628189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,1024,2,0,0.07653866708278656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,1024,4,0,0.04321066538492838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,1024,64,0,0.027295999228954315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,1024,1,0,0.12794133027394614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,1536,16,0,0.03649600098530451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,1536,4,0,0.0668746680021286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,1536,32,0,0.03554133325815201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,1536,8,0,0.04093866546948751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,1536,64,0,0.03507733345031738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,1536,2,0,0.1285760005315145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,1536,1,0,0.23107733329137167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,1536,16,0,0.036474667489528656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,1536,32,0,0.035562666753927864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,1536,4,0,0.0665226678053538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,1536,8,0,0.04138133426507314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,1536,64,0,0.03507733345031738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,1536,2,0,0.12777599692344666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,1536,1,0,0.22738667329152426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,2048,8,0,0.05550399919350942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,2048,4,0,0.0999840001265208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,2048,16,0,0.0451200008392334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,2048,32,0,0.04318933188915253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,2048,64,0,0.04292266567548116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,2048,2,0,0.19378666083017984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,2048,8,0,0.05560533205668131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,2048,1,0,0.3513706525166829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,2048,4,0,0.10094400246938069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,2048,32,0,0.043477331598599754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,2048,16,0,0.044938668608665466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,2048,2,0,0.1949653426806132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,2048,64,0,0.042581334710121155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,2048,1,0,0.3577386538187663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,3072,8,0,0.10181333621342976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,3072,16,0,0.06478400031725566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,3072,32,0,0.05941333373387655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,3072,4,0,0.20801067352294922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,3072,64,0,0.05792533357938131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,3072,2,0,0.3737706740697225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,3072,8,0,0.10300800204277039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,3072,4,0,0.2069279948870341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,3072,16,0,0.064751997590065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,3072,32,0,0.05933333436648051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,3072,1,0,0.6909120082855225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,3072,64,0,0.0580213318268458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,3072,2,0,0.36904001235961914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,4096,32,0,0.07642666498819987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,4096,16,0,0.08693866928418477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,4096,8,0,0.15641066431999207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,4096,4,0,0.3315253257751465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,3072,1,0,0.6968746980031332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,4096,64,0,0.07357333103815715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,4096,2,0,0.6059306859970093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,4096,8,0,0.1573973298072815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,4096,16,0,0.0865226686000824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,4096,4,0,0.3304266730944316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,4096,32,0,0.07551466425259908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,4096,64,0,0.07342400153477986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,4096,2,0,0.5958453416824341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,4096,1,0,1.1577013333638508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,6144,32,0,0.11363733808199565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,6144,16,0,0.17566933234532675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,6144,8,0,0.3628213405609131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,6144,4,0,0.6563359896341959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,6144,64,0,0.10568533341089885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,4096,1,0,1.1528639793395996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,6144,2,0,1.2617653210957844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,6144,16,0,0.17696533600489298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,6144,8,0,0.3646080096562703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,6144,4,0,0.6556053161621094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,6144,64,0,0.10596266388893127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,6144,32,0,0.11416533589363098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,6144,2,0,1.2353013356526692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,8192,8,0,0.5998026529947916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,6144,1,0,2.4268746376037598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,8192,16,0,0.2776426672935486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,8192,4,0,1.086352030436198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,8192,32,0,0.15037332971890768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,8192,64,0,0.14100799957911173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,6144,1,0,2.3912693659464517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,8192,2,0,2.1251093546549478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,8192,8,0,0.5928693215052286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,8192,16,0,0.27481067180633545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,8192,4,0,1.1107839743296306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,8192,32,0,0.14934933185577393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,8192,64,0,0.14111999670664468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,8192,2,0,2.0877493222554526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,10240,8,0,0.881114641825358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,8192,1,0,4.2391360600789385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,10240,32,0,0.2336906592051188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,10240,4,0,1.6599040031433105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,10240,64,0,0.17690666516621908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,10240,16,0,0.49500266710917157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,10240,8,0,0.876960039138794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,10240,2,0,3.2555999755859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,8192,1,0,4.23195743560791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,10240,32,0,0.2367253303527832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,10240,64,0,0.17735467354456583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,10240,16,0,0.4960319995880127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,10240,4,0,1.6502559979756672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,10240,2,0,3.353621482849121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,12288,8,0,1.2247467041015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,12288,16,0,0.6846720377604166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,10240,1,0,6.54534912109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,12288,4,0,2.310512065887451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,12288,32,0,0.3256640036900838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,12288,64,0,0.210698664188385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,12288,2,0,4.751119931538899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,12288,8,0,1.2237173716227214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,10240,1,0,6.469562530517578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,12288,32,0,0.32782934109369916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,12288,64,0,0.21261866887410483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,12288,16,0,0.6816266377766927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,12288,4,0,2.3148533503214517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,16384,8,0,2.076730728149414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,12288,2,0,4.6887359619140625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,12288,1,0,10.008911768595377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,16384,4,0,4.124725341796875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,16384,16,0,1.1228000322977703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,16384,64,0,0.2805386583010356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,16384,32,0,0.5197973251342773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,16384,8,0,2.07259194056193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,16384,2,0,8.3503786722819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,12288,1,0,9.538096110026041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,16384,64,0,0.27616532643636066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,16384,32,0,0.5272266864776611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,16384,16,0,1.1189440091451008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,16384,4,0,4.076800028483073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,16,1,0,0.018207999567190807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,16,8,0,0.013386666774749756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,16,4,0,0.014032000054915747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,16,2,0,0.014858666807413101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,16,16,0,0.012965332716703415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,16,32,0,0.012666666259368261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,16,64,0,0.012629333883523941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,16,1,0,0.018053332964579265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,16,2,0,0.014949332922697067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,16,4,0,0.013776000589132309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,16,16,0,0.013370666652917862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,16,8,0,0.01851733277241389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,16,32,0,0.012634667257467905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,16,64,0,0.012821332861979803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,32,1,0,0.01868266612291336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,32,2,0,0.015082667271296183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,32,4,0,0.014181333283583323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,32,8,0,0.013679999858140945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,32,16,0,0.013503999759753546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,32,32,0,0.024453334510326385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,16384,2,0,8.331781387329102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,32,64,0,0.012778667112191519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,32,4,0,0.013957332819700241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,32,1,0,0.01854933301607768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,32,2,0,0.014917333920796713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,32,8,0,0.014997333288192749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,32,16,0,0.013343999783198038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,32,32,0,0.012784000486135483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,32,64,0,0.013045333325862885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,64,4,0,0.014752000570297241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,64,1,0,0.02075733368595441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,64,2,0,0.03474666674931844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,16384,1,0,16.972309112548828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,64,8,0,0.014362666755914688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,64,16,0,0.014266667266686758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,64,32,0,0.024117333193620045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,64,64,0,0.013760000467300415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,64,1,0,0.02089066555102666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,64,2,0,0.016688000410795212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,64,4,0,0.01471466695268949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,64,16,0,0.016010666886965435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,64,8,0,0.014405333747466406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,64,64,0,0.014197333405415217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,64,32,0,0.014335999886194864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,128,1,0,0.03145066648721695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,128,2,0,0.019482667247454327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,128,4,0,0.01657066618402799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,128,8,0,0.01562133307258288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,128,16,0,0.015599999576807022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,128,32,0,0.015861333658297855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,128,64,0,0.015087999403476715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,128,1,0,0.03137599925200144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,128,8,0,0.020842666427294414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,128,2,0,0.019440000255902607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,128,16,0,0.015754666179418564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,128,4,0,0.016496000190575916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,128,32,0,0.015626666446526844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,128,64,0,0.015301333119471868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,256,8,0,0.018053332964579265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,256,4,0,0.020015999674797058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,256,1,0,0.061792001128196716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,256,16,0,0.01836266616980235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,256,2,0,0.031957333286603294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,256,32,0,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,16384,1,0,16.90789286295573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,256,64,0,0.016021333634853363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,256,8,0,0.018197332819302876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,256,4,0,0.019925333559513092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,256,2,0,0.032229334115982056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,256,16,0,0.0436160018046697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,256,1,0,0.06063466767470042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,256,64,0,0.016106666376193363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,256,32,0,0.01735466718673706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,512,16,0,0.021370666722456615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,512,4,0,0.03496533383925756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,512,8,0,0.022287999590237934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,512,64,0,0.020389333367347717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,512,32,0,0.020527999848127365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,512,2,0,0.06868800024191539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,512,1,0,0.10030399759610494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,512,32,0,0.020474666108687718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,512,16,0,0.021104000508785248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,512,64,0,0.020261333634455998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,512,4,0,0.03503466645876566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,512,2,0,0.06942933301130931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,512,8,0,0.022650666534900665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,512,1,0,0.10085866848627727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,1024,16,0,0.03075733284155528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,1024,8,0,0.04233600199222565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,1024,32,0,0.028981332977612812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,1024,64,0,0.028309332827727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,1024,4,0,0.07733333110809326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,1024,2,0,0.13058132926623026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,1024,16,0,0.030421334008375805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,1024,8,0,0.042453333735466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,1024,32,0,0.028837333122889202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,1024,4,0,0.07810133198897044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,1024,1,0,0.23672000567118326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,1024,2,0,0.12986133495966592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,1024,64,0,0.028192001084486645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,1024,1,0,0.23669334252675375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,1536,32,0,0.03781333317359289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,1536,16,0,0.04225599765777588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,1536,8,0,0.0677706648906072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,1536,4,0,0.13051733374595642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,1536,64,0,0.03636800001064936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,1536,2,0,0.23215999205907187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,1536,8,0,0.06796266635258992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,1536,32,0,0.0371573343873024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,1536,4,0,0.13009599844614664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,1536,16,0,0.04234666625658671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,1536,1,0,0.43006400267283124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,1536,64,0,0.03628266602754593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,1536,2,0,0.2286240061124166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,2048,16,0,0.0574239989121755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,2048,8,0,0.10292266805966695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,2048,32,0,0.04736533264319102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,2048,4,0,0.19758933782577515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,2048,64,0,0.04433600107828776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,1536,1,0,0.4270399808883667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,2048,2,0,0.35950934886932373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,2048,16,0,0.05773866673310598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,2048,8,0,0.10321600238482158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,2048,4,0,0.19818667570749918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,2048,32,0,0.050288001696268715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,2048,64,0,0.04438399771849314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,2048,2,0,0.3555146853129069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,2048,1,0,0.6963253021240234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,3072,16,0,0.10560533404350281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,3072,8,0,0.21040532986323038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,3072,4,0,0.3742453257242839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,3072,32,0,0.06805333495140076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,3072,64,0,0.06159999966621399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,2048,1,0,0.6854453086853027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,3072,2,0,0.7034613291422526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,3072,16,0,0.10664000113805135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,3072,8,0,0.20971733331680298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,3072,32,0,0.06771733363469441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,3072,4,0,0.3764266570409139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,3072,64,0,0.062277331948280334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,3072,2,0,0.7081920305887858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,4096,8,0,0.3311946590741475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,4096,16,0,0.16446933150291443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,4096,4,0,0.6171146631240845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,3072,1,0,1.3709707260131836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,4096,32,0,0.09058133761088054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,4096,64,0,0.0814879983663559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,4096,2,0,1.1618719895680745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,3072,1,0,1.3568159739176433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,4096,8,0,0.33284799257914227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,4096,16,0,0.16354667147000632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,4096,4,0,0.606053352355957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,4096,32,0,0.08930133779843648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,4096,64,0,0.08149866759777069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,4096,2,0,1.165610631306966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,6144,8,0,0.6633119980494181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,6144,16,0,0.3699520031611125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,6144,32,0,0.1854133407274882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,4096,1,0,2.2597813606262207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,6144,4,0,1.2362613677978516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,6144,64,0,0.11894933382670085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,6144,8,0,0.6619199911753336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,4096,1,0,2.3206186294555664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,6144,2,0,2.4185546239217124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,6144,64,0,0.11843199531237285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,6144,4,0,1.2314453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,6144,32,0,0.18483734130859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,6144,16,0,0.37277865409851074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,8192,8,0,1.104309320449829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,6144,2,0,2.409066677093506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,6144,1,0,5.095370610555013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,8192,32,0,0.28576000531514484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,8192,4,0,2.1487733523050943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,8192,16,0,0.6000693241755167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,8192,64,0,0.15970666209856668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,8192,8,0,1.1010133425394695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,8192,2,0,4.2262827555338545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,6144,1,0,5.0106401443481445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,8192,4,0,2.1162986755371094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,8192,32,0,0.28571200370788574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,8192,16,0,0.5989333391189575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,8192,64,0,0.15760533014933267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,10240,8,0,1.6607680320739746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,8192,2,0,4.220133463541667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,10240,32,0,0.5153066714604696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,10240,16,0,0.8942293326059977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,10240,4,0,3.3203201293945312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,10240,64,0,0.24790932734807333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,8192,1,0,8.669071833292643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,10240,2,0,6.602869033813477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,10240,4,0,3.3286399841308594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,10240,8,0,1.6582934061686199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,8192,1,0,8.645829518636068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,10240,64,0,0.246394673983256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,10240,16,0,0.8843733469645182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,10240,32,0,0.5058773358662924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,10240,2,0,6.550591786702474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,12288,8,0,2.3747785886128745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,12288,16,0,1.2371466954549153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,12288,32,0,0.6962347030639648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,12288,64,0,0.3362933397293091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,12288,4,0,4.733008066813151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,10240,1,0,13.45315170288086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,12288,8,0,2.3351093928019204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,10240,1,0,13.555951436360678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,12288,2,0,9.611514409383139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,12288,16,0,1.2389386494954426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,12288,4,0,4.650010744730632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,12288,32,0,0.6991093158721924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,12288,64,0,0.33830400307973224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,16384,8,0,4.026447931925456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,12288,2,0,9.529589335123697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,16384,16,0,2.0987040201822915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,16384,32,0,1.1411200364430745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,16384,4,0,8.408778508504232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,16384,64,0,0.5364746650060018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,12288,1,0,19.08072026570638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,12288,1,0,19.772149403889973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,16384,8,0,4.0547841389973955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,16384,2,0,16.68450673421224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,16384,4,0,8.33784548441569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,16384,64,0,0.5326559940973917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,16384,32,0,1.136250654856364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,16,1,0,0.025120000044504803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,16,2,0,0.018207999567190807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,16384,16,0,2.1320746739705405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,16,4,0,0.014384000251690546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,16,8,0,0.01392000044385592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,16,16,0,0.01328533391157786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,16,32,0,0.013237333546082178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,16,64,0,0.012586666891972223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,16,2,0,0.018229333062966663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,16,1,0,0.02499199906984965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,16,4,0,0.014501333236694336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,16,8,0,0.013749333719412485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,16,16,0,0.013503999759753546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,16,32,0,0.013237333546082178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,16,64,0,0.012826666235923767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,32,1,0,0.025311999022960663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,32,2,0,0.018602666755517323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,32,4,0,0.014677333335081736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,32,8,0,0.014271999398867289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,32,16,0,0.013621332744757334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,32,32,0,0.01309866706530253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,32,64,0,0.012965332716703415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,32,1,0,0.02554133286078771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,32,2,0,0.01842133328318596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,32,4,0,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,32,8,0,0.0144213338692983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,32,16,0,0.013631999492645264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,32,32,0,0.012986666212479273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,32,64,0,0.012693333129088083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,64,4,0,0.01657066618402799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,64,2,0,0.020714666694402695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,64,1,0,0.036533333361148834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,64,8,0,0.014757333944241205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,16384,2,0,16.712788899739582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,64,16,0,0.01700266698996226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,64,32,0,0.014314666390419006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,64,64,0,0.014186666657527288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,64,1,0,0.03262399882078171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,64,4,0,0.01661866654952367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,64,16,0,0.014901333798964819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,64,2,0,0.020666666328907013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,64,8,0,0.014826666563749313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,64,32,0,0.014261333892742792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,64,64,0,0.013962666193644205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,128,2,0,0.03156800071398417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,128,4,0,0.019685332973798115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,128,8,0,0.016986666868130367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,128,32,0,0.015685333559910457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,128,16,0,0.01587733378012975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,128,1,0,0.05573866764704386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,128,64,0,0.01509333277742068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,128,1,0,0.05585599939028422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,128,2,0,0.03161066770553589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,128,16,0,0.015850666910409927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,128,4,0,0.01987733319401741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,128,8,0,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,128,32,0,0.015791999797026317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,128,64,0,0.015226667126019796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,256,8,0,0.01956266661485036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,256,4,0,0.033285332222779594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,256,2,0,0.06075733403364817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,256,16,0,0.018298666924238205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,256,32,0,0.01724799970785777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,256,1,0,0.09384533762931824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,256,64,0,0.016986666868130367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,256,2,0,0.06066133578618368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,256,16,0,0.01838933303952217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,256,8,0,0.024746666351954143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,256,4,0,0.033157333731651306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,256,1,0,0.09368000427881877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,256,32,0,0.01802666609485944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,256,64,0,0.016629333297411602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,16384,1,0,33.71291859944662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,512,16,0,0.023018665611743927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,512,8,0,0.03633599976698557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,512,4,0,0.07022933165232341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,512,2,0,0.09992000460624695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,512,32,0,0.021829334398110706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,512,64,0,0.021216000119845074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,512,1,0,0.18315200010935465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,512,4,0,0.070783997575442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,512,8,0,0.035642666121323906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,512,16,0,0.023183998962243397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,512,2,0,0.10037333766619365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,512,32,0,0.021759999295075733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,512,64,0,0.021183999876181286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,512,1,0,0.18287465969721475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,1024,16,0,0.04436799883842468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,1024,8,0,0.08070933322111766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,1024,4,0,0.13127467036247253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,1024,32,0,0.03180799881617228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,1024,64,0,0.029722665747006733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,1024,2,0,0.23863466580708823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,1024,8,0,0.08018666505813599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,1024,16,0,0.044437333941459656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,1024,4,0,0.1323093374570211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,1024,32,0,0.03251733382542928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,1024,64,0,0.02976000060637792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,1024,1,0,0.4517013231913249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,1024,2,0,0.23622934023539224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,1536,16,0,0.07052800059318542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,1536,8,0,0.1338933308919271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,1024,1,0,0.4564106861750285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,16384,1,0,33.828651428222656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,1536,64,0,0.04012800008058548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,1536,4,0,0.23281067609786987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,1536,32,0,0.04488533238569895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,1536,16,0,0.07072000205516815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,1536,8,0,0.13236799836158752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,1536,2,0,0.42952533562978107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,1536,4,0,0.23376532395680746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,1536,32,0,0.044810667634010315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,1536,64,0,0.03946666667858759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,1536,2,0,0.4328480164210002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,1536,1,0,0.8457866509755453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,2048,8,0,0.2001439929008484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,2048,16,0,0.10796800255775452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,2048,32,0,0.06069866816202799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,2048,4,0,0.3589973449707031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,2048,64,0,0.051882664362589516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,1536,1,0,0.8559733231862386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,2048,2,0,0.6929333209991455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,2048,8,0,0.20046399037043253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,2048,16,0,0.10748266180356343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,2048,4,0,0.3591839869817098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,2048,32,0,0.06098666787147522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,2048,64,0,0.051882664362589516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,2048,2,0,0.6929386456807455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,2048,1,0,1.3392640749613445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,3072,8,0,0.3781546751658122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,3072,16,0,0.21719467639923096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,3072,32,0,0.11238933602968852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,3072,64,0,0.07307200133800507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,3072,4,0,0.7130666573842367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,2048,1,0,1.3479359944661458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,3072,8,0,0.3792959849039714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,3072,2,0,1.3627893129984539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,3072,16,0,0.22083733479181925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,3072,32,0,0.11435733238855998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,3072,4,0,0.7087626457214355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,3072,64,0,0.07211733361085255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,3072,2,0,1.3605920473734539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,4096,8,0,0.6090986728668213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,4096,32,0,0.16993065675099692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,4096,16,0,0.3369973500569661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,4096,4,0,1.1695573329925537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,4096,64,0,0.09731200337409973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,3072,1,0,2.7679147720336914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,4096,2,0,2.294933319091797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,4096,8,0,0.6234240134557089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,3072,1,0,2.7397120793660483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,4096,64,0,0.09668266773223877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,4096,32,0,0.1732693314552307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,4096,16,0,0.3404159943262736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,4096,4,0,1.1579786936442058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,4096,2,0,2.284976005554199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,6144,8,0,1.2463786602020264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,4096,1,0,4.612266540527344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,6144,32,0,0.37724800904591876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,6144,16,0,0.668282667795817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,6144,4,0,2.4646132787068686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,6144,64,0,0.20001065731048584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,6144,8,0,1.2578879992167156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,6144,2,0,5.186842600504558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,4096,1,0,4.494736035664876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,6144,16,0,0.6717226505279541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,6144,4,0,2.438389301300049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,6144,32,0,0.3932533264160156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,6144,64,0,0.20016533136367798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,6144,2,0,5.078799883524577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,8192,8,0,2.1135786374409995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,8192,32,0,0.6062453190485636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,8192,16,0,1.1150346597035725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,8192,64,0,0.29922666152318317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,8192,4,0,4.231733322143555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,6144,1,0,10.329418818155924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,8192,8,0,2.132218678792318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,8192,2,0,8.671466827392578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,8192,4,0,4.2327626546223955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,8192,32,0,0.6171733140945435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,8192,16,0,1.119808038075765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,6144,1,0,10.315408070882162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,16,1,0,0.03752533346414566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,16,2,0,0.02483733246723811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,16,4,0,0.017759999881188076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,16,8,0,0.014933332800865173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,8192,64,0,0.30341867605845135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,16,16,0,0.013786666095256805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,16,32,0,0.013429333766301474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,16,64,0,0.013280000537633896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,16,2,0,0.024714666108290356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,16,1,0,0.037434667348861694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,16,4,0,0.01764800027012825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,16,8,0,0.014922666052977243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,16,16,0,0.013909333695967993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,16,32,0,0.014592000593741735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,16,64,0,0.013045333325862885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,32,1,0,0.040922666589419045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,32,4,0,0.018245333184798557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,32,2,0,0.024864000578721363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,32,8,0,0.014837333311637243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,32,16,0,0.014111999422311783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,32,32,0,0.013834666460752487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,32,64,0,0.013397333522637686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,32,1,0,0.040965333580970764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,32,2,0,0.025173333783944447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,32,8,0,0.014975999792416891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,32,4,0,0.018218666315078735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,32,16,0,0.01393066719174385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,32,32,0,0.013936000565687815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,32,64,0,0.013429333766301474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,64,2,0,0.032645332316557564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,64,1,0,0.05406933526198069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,8192,2,0,8.657951990763346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,64,4,0,0.020655999581019085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,64,8,0,0.016666666915019352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,64,16,0,0.014858666807413101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,64,32,0,0.015173333386580149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,64,64,0,0.01452800010641416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,64,2,0,0.033146666983763375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,64,1,0,0.05413866539796194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,64,8,0,0.016458666572968166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,64,16,0,0.01482133318980535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,64,4,0,0.03669866671164831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,64,32,0,0.014752000570297241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,64,64,0,0.014463999619086584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,128,4,0,0.03299733251333237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,128,8,0,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,128,16,0,0.01699200024207433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,128,2,0,0.05589333176612854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,128,1,0,0.09937600294748943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,128,32,0,0.01618133361140887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,128,64,0,0.015850666910409927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,128,16,0,0.01701333373785019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,128,8,0,0.02107200026512146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,128,4,0,0.03752533346414566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,128,2,0,0.055914665261904396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,128,1,0,0.09912533561388652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,128,32,0,0.016410666207472484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,128,64,0,0.015568000574906668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,256,8,0,0.03324266771475474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,8192,1,0,17.69273630777995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,256,16,0,0.020879998803138733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,256,4,0,0.06274133423964183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,256,2,0,0.09435733159383138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,256,32,0,0.018672000616788864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,256,1,0,0.17015467087427774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,256,64,0,0.02004266654451688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,256,4,0,0.06297599772612254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,256,8,0,0.03363200028737386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,256,16,0,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,256,2,0,0.09532266855239868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,256,32,0,0.018944000204404194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,256,64,0,0.01773333301146825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,256,1,0,0.1699946721394857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,512,8,0,0.0734506646792094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,512,16,0,0.0386559988061587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,512,32,0,0.024293333292007446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,512,4,0,0.1034879982471466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,512,64,0,0.022570667167504627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,512,2,0,0.1845653255780538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,512,8,0,0.07375999788443248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,512,4,0,0.10259733597437541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,512,16,0,0.037952000896135964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,512,32,0,0.024586667617162068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,512,64,0,0.022645334402720135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,512,1,0,0.34458665053049725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,512,2,0,0.18352532386779785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,1024,16,0,0.08528000116348267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,512,1,0,0.3418826659520467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,1024,8,0,0.1362773378690084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,1024,32,0,0.048309331138928734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,1024,64,0,0.0365280012289683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,1024,4,0,0.24303466081619263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,1024,2,0,0.45346665382385254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,1024,8,0,0.13665067156155905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,8192,1,0,17.854352315266926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,1024,4,0,0.24155733982721964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,1024,16,0,0.08416533470153809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,1024,64,0,0.037871999045213066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,1024,32,0,0.048384000857671104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,1024,2,0,0.45103998978932697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,1024,1,0,0.8972373008728027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,1536,8,0,0.2392959992090861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,1536,16,0,0.14039466778437296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,1536,32,0,0.07695466776688893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,1536,64,0,0.05049600203831991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,1536,4,0,0.43856533368428546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,1024,1,0,0.8866133689880371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,1536,16,0,0.13874133427937826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,1536,8,0,0.23918932676315308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,1536,2,0,0.8461600144704183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,1536,64,0,0.05100800096988678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,1536,32,0,0.07683200140794118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,1536,4,0,0.44150400161743164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,1536,2,0,0.8607626756032308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,2048,8,0,0.3712906837463379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,2048,16,0,0.20801067352294922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,2048,32,0,0.11506133278210957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,2048,4,0,0.701136032740275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,2048,64,0,0.0673173318306605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,1536,1,0,1.6537119547526042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,1536,1,0,1.6557493209838867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,2048,2,0,1.3555359840393066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,2048,8,0,0.36743466059366864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,2048,32,0,0.11543466647466023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,2048,4,0,0.7069546381632487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,2048,64,0,0.0681386689345042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,2048,16,0,0.20735466480255127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,2048,2,0,1.3620266914367676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,3072,8,0,0.7157813707987467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,3072,16,0,0.3890933195749919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,3072,32,0,0.22669333219528198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,3072,4,0,1.3722507158915203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,3072,64,0,0.12336533268292744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,2048,1,0,2.7245814005533853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,3072,8,0,0.719808022181193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,3072,2,0,2.7696746190389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,2048,1,0,2.7277654012044272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,3072,4,0,1.3890453974405925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,3072,16,0,0.38942933082580566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,3072,32,0,0.22635199626286825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,3072,64,0,0.12989866733551025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,3072,2,0,2.7107998530069985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,4096,8,0,1.1871999899546306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,4096,32,0,0.3529226779937744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,4096,16,0,0.6213653484980265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,4096,64,0,0.18531199296315512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,4096,4,0,2.341200033823649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,3072,1,0,5.507855733235677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,4096,8,0,1.1867307027180989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,3072,1,0,5.504112243652344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,4096,2,0,4.721071879069011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,4096,4,0,2.3450133005777993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,4096,64,0,0.1918026606241862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,4096,32,0,0.3505493402481079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,16,1,0,0.06080533564090729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,4096,16,0,0.6272000074386597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,16,2,0,0.035429333647092186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,16,4,0,0.02369600037733714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,16,8,0,0.017877332866191864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,16,16,0,0.015002666662136713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,16,32,0,0.0136266661187013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,16,64,0,0.013295999417702356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,16,4,0,0.02382933348417282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,16,2,0,0.03807999938726425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,16,1,0,0.06073066592216492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,16,8,0,0.017690667261679966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,16,16,0,0.014885333677132925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,16,32,0,0.04422399898370107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,16,64,0,0.013365333278973898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,32,1,0,0.06870933373769124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,32,8,0,0.018101333330074947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,32,4,0,0.025125332176685333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,32,2,0,0.04108799993991852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,32,16,0,0.014975999792416891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,32,64,0,0.01381333296497663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,32,32,0,0.01413333291808764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,32,1,0,0.06843733290831248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,32,2,0,0.04085333396991094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,32,8,0,0.03726933399836222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,32,4,0,0.024933333198229473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,32,16,0,0.014720000326633453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,32,32,0,0.014053333550691605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,32,64,0,0.013642666240533194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,4096,2,0,4.742512067159017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,64,2,0,0.05455466608206431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,64,8,0,0.020992000897725422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,64,4,0,0.03342399994532267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,64,1,0,0.09629333019256592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,64,16,0,0.01716800034046173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,64,32,0,0.015018666783968607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,64,64,0,0.014565333724021912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,64,4,0,0.03359466542800268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,64,2,0,0.054885332783063255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,64,8,0,0.03473600000143051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,64,16,0,0.016943999876578648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,64,1,0,0.09609599908192952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,64,32,0,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,64,64,0,0.014762666076421738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,128,8,0,0.033514666060606636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,128,16,0,0.02102400114138921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,128,2,0,0.0999840001265208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,128,4,0,0.05769066512584686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,128,32,0,0.017711999515692394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,128,64,0,0.017045332739750545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,128,1,0,0.1845546762148539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,128,8,0,0.03399466723203659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,128,2,0,0.09990400075912476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,128,4,0,0.05806933343410492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,128,16,0,0.020175999651352566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,128,64,0,0.016789333273967106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,128,32,0,0.020831999679406483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,128,1,0,0.18224000930786133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,4096,1,0,9.623162587483725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,256,16,0,0.036848001182079315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,256,32,0,0.022128000855445862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,256,8,0,0.06422399977842967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,256,4,0,0.09598933657010396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,256,64,0,0.019813333948453266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,256,2,0,0.17243200540542603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,256,16,0,0.03752533346414566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,256,8,0,0.06452799836794536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,256,4,0,0.09712533156077068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,256,1,0,0.32128000259399414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,256,32,0,0.021850667893886566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,256,2,0,0.17180800437927246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,256,64,0,0.019498666127522785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,512,16,0,0.0787360022465388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,256,1,0,0.321888009707133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,512,8,0,0.10611733794212341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,512,32,0,0.04301866888999939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,512,4,0,0.18737600247065225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,512,64,0,0.030784000953038532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,512,2,0,0.3468746741612752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,512,8,0,0.10683199763298035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,512,4,0,0.1872319976488749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,512,16,0,0.07875733574231465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,512,32,0,0.04339733223120371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,512,64,0,0.030293333033720653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,512,2,0,0.348960002263387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,4096,1,0,9.519194920857748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,512,1,0,0.6638933420181274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,1024,8,0,0.2518293261528015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,1024,4,0,0.46137599150339764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,1024,32,0,0.09205866853396098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,512,1,0,0.6646186510721842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,1024,64,0,0.05659199754397074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,1024,16,0,0.14467733105023703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,1024,16,0,0.14360533157984415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,1024,8,0,0.25125332673390705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,1024,2,0,0.8938506444295248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,1024,32,0,0.09270933270454407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,1024,64,0,0.05696000158786774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,1024,4,0,0.4596586624781291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,1536,8,0,0.4475466807683309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,1024,2,0,0.8845653533935547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,1536,4,0,0.8729546864827474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,1536,16,0,0.24802666902542114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,1536,32,0,0.14867200454076132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,1536,64,0,0.087226668993632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,1024,1,0,1.7772480646769206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,1536,2,0,1.663647969563802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,1536,8,0,0.45023465156555176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,1536,32,0,0.14964800079663595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,1024,1,0,1.7706774075826008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,1536,16,0,0.24860266844431558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,1536,4,0,0.8715413411458334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,1536,64,0,0.09312533338864644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,2048,8,0,0.7124319871266683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,1536,2,0,1.666927973429362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,2048,32,0,0.21917333205540976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,1536,1,0,3.372778574625651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,2048,16,0,0.3835200071334839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,2048,4,0,1.3957120577494304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,2048,64,0,0.1270133356253306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,2048,8,0,0.7063946723937988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,2048,16,0,0.3787200053532918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,2048,2,0,2.6954558690389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,1536,1,0,3.405872027079264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,2048,4,0,1.3939253489176433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,16,2,0,0.06272533535957336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,2048,32,0,0.21965332825978598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,16,1,0,0.11360533038775127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,2048,64,0,0.13548800349235535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,16,4,0,0.035045333206653595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,16,8,0,0.02367466688156128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,16,16,0,0.017653333644072216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,16,64,0,0.013658666362365087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,16,32,0,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,16,2,0,0.06137066582838694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,16,1,0,0.10995200276374817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,16,4,0,0.0354666660229365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,16,16,0,0.017605333278576534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,16,8,0,0.0240639994541804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,16,32,0,0.014511999984582266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,16,64,0,0.013866666704416275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,32,4,0,0.041690667470296226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,32,8,0,0.0249439999461174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,32,2,0,0.0693333347638448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,32,1,0,0.1323040028413137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,32,32,0,0.015077333897352219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,32,16,0,0.01821333294113477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,32,64,0,0.014181333283583323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,32,4,0,0.04091199984153112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,32,2,0,0.06970133384068807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,32,1,0,0.13150399923324585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,32,16,0,0.018186666071414948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,32,8,0,0.024864000578721363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,32,32,0,0.015008000036080679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,2048,2,0,2.7272373835245767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,32,64,0,0.014293332894643148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,64,16,0,0.02195200075705846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,64,8,0,0.03621333340803782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,64,4,0,0.05600533386071523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,64,64,0,0.01594666639963786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,64,32,0,0.017509333789348602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,64,2,0,0.09768533706665039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,64,1,0,0.17851734161376953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,64,4,0,0.056426664193471275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,64,2,0,0.09729599952697754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,64,8,0,0.03488533447186152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,64,16,0,0.02145066608985265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,64,32,0,0.017722666263580322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,64,1,0,0.17845332622528076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,64,64,0,0.0161013330022494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,2048,1,0,5.674437204996745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,128,16,0,0.035605333745479584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,128,8,0,0.05961599946022034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,128,32,0,0.02202133337656657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,128,4,0,0.10229333241780598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,128,64,0,0.018895999838908512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,128,2,0,0.18348799149195352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,128,8,0,0.06065066655476888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,128,16,0,0.03555733213822047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,128,4,0,0.10186133782068889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,128,32,0,0.021727999051411945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,128,2,0,0.18807466824849448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,128,64,0,0.018496000518401463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,128,1,0,0.3461386760075887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,128,1,0,0.3473386764526367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,256,8,0,0.10052800178527832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,256,16,0,0.06997333467006683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,256,4,0,0.17484267552693686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,256,64,0,0.026533332963784535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,256,32,0,0.04738133152325948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,256,8,0,0.10101333260536194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,256,2,0,0.32369067271550495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,256,4,0,0.1758240063985189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,256,16,0,0.0686773310105006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,256,32,0,0.042026668787002563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,2048,1,0,5.66111437479655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,256,64,0,0.026767998933792114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,256,2,0,0.3237973252932231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,256,1,0,0.6218826770782471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,512,16,0,0.114138662815094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,512,32,0,0.08718400200208028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,512,8,0,0.1939786672592163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,512,64,0,0.0517493337392807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,512,4,0,0.35464000701904297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,256,1,0,0.623029351234436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,512,8,0,0.19830399751663208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,512,2,0,0.6736640135447184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,512,4,0,0.35336001714070636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,512,32,0,0.08736532926559448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,512,16,0,0.11428800225257874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,512,64,0,0.05300800005594889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,512,2,0,0.6693706512451172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,1024,8,0,0.47605868180592853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,1024,16,0,0.26103466749191284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,512,1,0,1.3579039573669434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,1024,64,0,0.10587199529012044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,1024,32,0,0.1578879952430725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,1024,4,0,0.8979679743448893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,512,1,0,1.3438773155212402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,1024,16,0,0.26262933015823364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,1024,32,0,0.15646933515866598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,1024,8,0,0.4737386703491211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,1024,2,0,1.7959839502970378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,1024,64,0,0.10640533765157063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,64,16,1,0,0.24882133801778158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,1024,4,0,0.9079626401265463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,16,2,0,0.1143893301486969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,16,4,0,0.06353599826494853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,16,8,0,0.03527999917666117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,16,16,0,0.0239680012067159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,16,32,0,0.017781333376963932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,16,64,0,0.014741333822409311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,16,8,0,0.03514133393764496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,16,2,0,0.11017066240310669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,16,4,0,0.061903998255729675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,64,16,1,0,0.2485439976056417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,16,16,0,0.023706667125225067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,16,32,0,0.017994667092959087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,16,64,0,0.014848000059525171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,32,8,0,0.042405332128206887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,32,4,0,0.0699786643187205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,32,2,0,0.1323946714401245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,32,16,0,0.025290665527184803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,32,32,0,0.018805333723624546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,64,32,1,0,0.2637653350830078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,32,64,0,0.015637333194414776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,32,4,0,0.07050133248170216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,32,8,0,0.04327466587225596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,32,2,0,0.13275200128555298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,32,16,0,0.02500266581773758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,64,32,1,0,0.2641706665356954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,32,32,0,0.01868266612291336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,32,64,0,0.01563199982047081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,1024,2,0,1.7805387179056804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,64,8,0,0.057631999254226685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,64,4,0,0.10267200072606404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,64,16,0,0.03858133405447006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,64,64,0,0.018325333793958027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,64,32,0,0.02232533444960912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,64,2,0,0.1821440060933431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,64,8,0,0.05819199979305267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,64,64,1,0,0.3429226477940877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,64,4,0,0.10216533144315083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,64,16,0,0.03692800054947535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,64,2,0,0.18273067474365234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,64,32,0,0.022437334060668945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,1024,1,0,3.6111574172973633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,64,64,0,0.0183146670460701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,64,64,1,0,0.3422400156656901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,128,8,0,0.10795199871063232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,128,16,0,0.06418133278687795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,128,4,0,0.1904053290685018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,128,32,0,0.04009066770474116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,128,64,0,0.027098665634791057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,128,2,0,0.3511413335800171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,128,8,0,0.1074026624361674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,128,16,0,0.06410133341948192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,128,4,0,0.18976000944773355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,128,32,0,0.04030933231115341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,128,64,0,0.02682666728893916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,128,2,0,0.3521813154220581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,1024,1,0,3.625568072001139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,64,128,1,0,0.6740106741587321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,256,8,0,0.18212799231211343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,256,16,0,0.11173333724339803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,256,32,0,0.08433600266774495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,256,64,0,0.05133333305517832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,256,4,0,0.3302133282025655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,64,128,1,0,0.6717013518015543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,256,16,0,0.10872000455856323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,256,8,0,0.18231467405954996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,256,2,0,0.6281599998474121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,256,32,0,0.07741333544254303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,256,64,0,0.050570666790008545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,256,4,0,0.33051733175913495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,256,2,0,0.6279413302739462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,512,8,0,0.3645973205566406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,512,16,0,0.20814400911331177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,64,256,1,0,1.2683146794637044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,512,32,0,0.12786133090655008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,512,64,0,0.10088533163070679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,512,4,0,0.6874346733093262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,64,256,1,0,1.2667199770609539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,512,8,0,0.36559998989105225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,512,16,0,0.20816532770792642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,512,32,0,0.12643733620643616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,512,64,0,0.10087999701499939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,16,2,0,0.25116799275080365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,512,4,0,0.6892320315043131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,128,16,1,0,0.5020320018132528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,16,8,0,0.063509335120519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,16,4,0,0.11434666315714519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,16,16,0,0.03681066632270813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,16,32,0,0.024298667907714844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,16,64,0,0.01815466706951459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,16,4,0,0.1107413371404012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,16,2,0,0.2514080007870992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,16,8,0,0.06225599845250448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,128,16,1,0,0.5047573248545328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,16,16,0,0.03566933423280716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,16,32,0,0.023893333971500397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,16,64,0,0.01826133330663045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,512,2,0,1.3495839436848958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,32,4,0,0.13546133041381836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,32,2,0,0.26341332991917926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,32,16,0,0.044480000933011375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,32,8,0,0.07593066493670146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,32,32,0,0.026799999177455902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,32,64,0,0.019610666980346043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,128,32,1,0,0.5261173248291016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,512,2,0,1.3630560239156086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,32,16,0,0.04752533137798309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,32,4,0,0.1341653366883596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,32,8,0,0.07258666555086772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,32,32,0,0.02741866558790207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,32,2,0,0.26580800612767536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,32,64,0,0.01952533299724261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,128,32,1,0,0.5301920175552368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,64,8,0,0.10467200477917989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,64,512,1,0,2.6822665532430015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,64,16,0,0.06326933205127716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,64,4,0,0.1904266675313314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,64,64,0,0.028031999866167705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,64,32,0,0.04101866732041041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,64,2,0,0.34808532396952313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,64,8,0,0.10421866178512573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,64,4,0,0.1928106745084127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,64,32,0,0.04111466556787491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,64,16,0,0.06341866652170818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,64,64,0,0.027903998891512554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,64,2,0,0.34674131870269775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,128,64,1,0,0.6699786980946859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,128,16,0,0.11400533715883891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,128,8,0,0.19476799170176187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,128,64,1,0,0.6703680356343588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,128,32,0,0.07307733098665874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,128,4,0,0.37460800011952716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,128,64,0,0.05019199848175049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,64,512,1,0,2.6828266779581704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,128,8,0,0.19504533211390176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,128,2,0,0.6849653720855713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,128,32,0,0.07277333239714305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,128,16,0,0.11520533760388692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,128,4,0,0.3732159932454427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,128,64,0,0.05171200136343638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,128,2,0,0.6862826347351074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,256,8,0,0.344709316889445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,128,128,1,0,1.3566932678222656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,256,32,0,0.12139733632405598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,256,16,0,0.19546133279800415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,256,64,0,0.09192533294359843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,256,4,0,0.6429386536280314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,128,128,1,0,1.3533867200215657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,256,8,0,0.3454986810684204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,256,16,0,0.2216106653213501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,256,32,0,0.12167466680208842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,256,64,0,0.09212266405423482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,256,4,0,0.6431359847386678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,256,2,0,1.2827413082122803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,256,16,4,0,0.25648534297943115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,256,16,2,0,0.5063039859135946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,256,16,32,0,0.037615999579429626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,256,16,8,0,0.11672533551851909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,256,16,16,0,0.0650079995393753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,256,16,1,0,1.02619202931722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,256,16,64,0,0.025445332129796345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,256,16,8,0,0.11292266845703125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,256,16,4,0,0.2584799925486247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,256,16,2,0,0.5078239838282267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,256,16,32,0,0.038890667259693146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,256,16,16,0,0.06467733283837636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,256,16,64,0,0.025418666501839954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,256,16,1,0,1.0073706309000652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,256,2,0,1.2840159734090169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,256,32,16,0,0.08185066779454549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,256,32,8,0,0.13634133338928223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,256,32,4,0,0.2688693404197693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,256,32,32,0,0.04868799944718679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,256,32,64,0,0.031130666534105938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,256,32,2,0,0.5338506698608398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,256,32,8,0,0.13850667079289755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,256,32,4,0,0.26551999648412067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,256,32,16,0,0.07787199815114339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,256,32,32,0,0.04880533119042715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,256,32,1,0,1.0896639823913574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,256,32,64,0,0.03183999905983607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,256,32,2,0,0.5326026678085327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,128,256,1,0,2.535914738972982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,256,64,16,0,0.11376532912254333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,256,64,8,0,0.19419733683268228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,256,32,1,0,1.0605706373850505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,256,64,32,0,0.07153599957625072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,256,64,64,0,0.049466664592425026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,256,64,4,0,0.3715466658274333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,256,64,8,0,0.1939093271891276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,256,64,2,0,0.6796320279439291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,256,64,4,0,0.3683733145395915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,256,64,16,0,0.1125386655330658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,256,64,32,0,0.07206400235493977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,256,64,64,0,0.05012266834576925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,256,64,2,0,0.6801866690317789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,128,256,1,0,2.538304011027018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,256,64,1,0,1.3432745933532715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,256,128,16,0,0.20951465765635172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,256,128,8,0,0.37063467502593994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,256,128,32,0,0.1384106675783793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,256,128,4,0,0.7104960282643636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,256,128,64,0,0.08681600292523702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,256,64,1,0,1.3426879247029622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,256,128,16,0,0.2116426626841227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,256,128,8,0,0.37331732114156085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,256,128,32,0,0.12866666913032532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,256,128,64,0,0.0869653324286143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,256,128,2,0,1.4017225901285808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,256,128,4,0,0.7109973430633545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,256,128,2,0,1.3803253173828125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,256,128,1,0,2.7183574040730796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,256,128,1,0,2.7108373641967773
