framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,16,1,0,0.016762666404247284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,16,4,0,0.015781333049138386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,16,8,0,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,16,2,0,0.016021333634853363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,16,16,0,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,16,32,0,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,16,64,0,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,16,128,0,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,16,1,0,0.01953599974513054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,16,2,0,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,16,4,0,0.0186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,16,8,0,0.018053332964579265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,16,16,0,0.018378666291634243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,16,32,0,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,16,64,0,0.018229333062966663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,16,128,0,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,32,1,0,0.017797333498795826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,32,2,0,0.016506666938463848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,32,4,0,0.015919999529918034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,32,8,0,0.015925332903862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,32,16,0,0.015749332805474598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,32,32,0,0.016048000504573185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,32,64,0,0.016517333686351776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,32,128,0,0.015520000209410986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,32,1,0,0.020117333779732387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,32,2,0,0.019962667177120846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,32,4,0,0.01934933289885521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,32,8,0,0.01937066639463107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,32,32,0,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,32,64,0,0.018320000420014065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,32,128,0,0.018687999496857326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,32,16,0,0.01859733338157336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,64,1,0,0.01952533299724261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,64,2,0,0.017632000148296356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,64,4,0,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,64,8,0,0.016762666404247284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,64,16,0,0.01642666632930438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,64,32,0,0.015520000209410986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,64,64,0,0.016271999726692837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,64,128,0,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,64,4,0,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,64,1,0,0.022128000855445862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,64,2,0,0.02274666726589203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,64,8,0,0.02082666630546252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,64,16,0,0.020154666155576706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,64,32,0,0.020362666497627895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,64,64,0,0.02042666698495547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,64,128,0,0.020069333414236706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,128,1,0,0.022255999346574146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,128,2,0,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,128,4,0,0.018378666291634243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,128,8,0,0.017935999979575474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,128,16,0,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,128,32,0,0.017349333812793095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,128,64,0,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,128,128,0,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,128,1,0,0.025573333104451496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,128,2,0,0.02207999924818675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,128,4,0,0.021664001047611237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,128,8,0,0.02141333371400833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,128,16,0,0.021690666675567627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,128,64,0,0.020629333953062694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,128,32,0,0.021274665991465252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,256,1,0,0.03492266684770584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,256,16,0,0.019850666324297588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,256,32,0,0.019978666057189304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,256,64,0,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,256,2,0,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,128,128,0,0.020053333292404812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,256,1,0,0.05147199829419454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,256,4,0,0.024746666351954143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,256,2,0,0.027514666318893433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,256,16,0,0.0229120006163915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,256,8,0,0.023333333432674408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,256,4,0,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,256,64,0,0.022730665902296703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,256,128,0,0.018768000106016796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,256,32,0,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,256,8,0,0.02046400060256322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,512,1,0,0.06150400141874949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,256,128,0,0.023242667317390442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,512,8,0,0.02480533222357432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,512,128,0,0.022405333817005157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,512,2,0,0.04123199979464213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,512,4,0,0.026250667870044708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,512,2,0,0.0576800008614858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,512,16,0,0.02369600037733714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,512,32,0,0.023605334262053173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,512,64,0,0.022997332115968067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,512,4,0,0.030741333961486816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,512,32,0,0.0272533322374026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,512,1,0,0.09482666850090027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,512,8,0,0.027765333652496338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,1024,2,0,0.0809440016746521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,1024,4,0,0.05426133175690969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,512,16,0,0.026693334182103474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,512,64,0,0.025968000292778015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,512,128,0,0.02626666675011317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,1024,1,0,0.13778666655222574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,1024,32,0,0.03146133323510488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,1024,64,0,0.03146133323510488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,1024,8,0,0.03383466601371765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,1024,16,0,0.03272533416748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,1024,4,0,0.06924800078074138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,1024,128,0,0.030586667358875275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,1024,8,0,0.03783999880154928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,1024,1,0,0.1954080065091451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,1024,64,0,0.034074666599432625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,1024,2,0,0.10986666878064473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,1536,1,0,0.2435306708017985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,1024,16,0,0.03442133218050003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,1024,128,0,0.03334933271010717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,1024,32,0,0.034314667185147606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,1536,4,0,0.08379200100898743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,1536,16,0,0.041034666200478874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,1536,2,0,0.1390506625175476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,1536,1,0,0.31669867038726807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,1536,8,0,0.05765333275000254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,1536,16,0,0.04248533149560293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,1536,32,0,0.040847999354203544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,1536,64,0,0.038831998904546104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,1536,4,0,0.1048959990342458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,1536,128,0,0.03934400031963984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,1536,64,0,0.04125866790612539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,1536,2,0,0.17717333634694418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,1536,8,0,0.06679466863473256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,2048,2,0,0.20934400955835977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,1536,32,0,0.358298659324646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,2048,8,0,0.08097599943478902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,1536,128,0,0.04046933352947235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,2048,1,0,0.3790880044301351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,2048,64,0,0.047413334250450134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,2048,4,0,0.12382400035858154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,2048,1,0,0.46697068214416504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,2048,16,0,0.050986667474110924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,2048,32,0,0.04872000217437744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,2048,8,0,0.09093333284060161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,2048,32,0,0.04864533245563507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,2048,128,0,0.04554666578769684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,2048,4,0,0.14417066176732382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,2048,2,0,0.25383466482162476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,3072,1,0,0.7503946622212728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,3072,2,0,0.3943680127461751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,2048,16,0,0.0545066644748052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,2048,128,0,0.049141332507133484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,2048,64,0,0.048714667558670044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,3072,64,0,0.06233066817124685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,3072,32,0,0.06438933312892914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,3072,4,0,0.2222773234049479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,3072,8,0,0.13763200243314108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,3072,16,0,0.09057066837946574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,3072,128,0,0.06233066817124685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,3072,2,0,0.4423786799112956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,3072,1,0,0.8395093282063802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,3072,4,0,0.24500799179077148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,3072,8,0,0.14573333660761514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,3072,16,0,0.0944106678167979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,3072,64,0,0.06126933296521505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,3072,32,0,0.06567466755708058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,3072,128,0,0.06204266846179962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,4096,4,0,0.35045866171518963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,4096,8,0,0.20586133003234863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,4096,1,0,1.296021302541097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,4096,1,0,1.221776008605957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,4096,2,0,0.6489013433456421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,4096,32,0,0.08269333342711131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,4096,16,0,0.13422933220863342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,4096,2,0,0.6780479749043783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,4096,64,0,0.0801333338022232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,4096,128,0,0.07784000039100647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,4096,4,0,0.37069865067799884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,4096,128,0,0.07442133128643036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,6144,4,0,0.6951200167338053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,4096,32,0,0.08288533488909404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,4096,8,0,0.21341333786646524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,4096,64,0,0.0757120003302892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,4096,16,0,0.13523200154304504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,6144,8,0,0.39292800426483154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,6144,2,0,1.3352640469868977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,6144,1,0,2.623840014139811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,6144,16,0,0.2351413369178772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,6144,2,0,1.296341339747111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,6144,16,0,0.22965333859125772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,6144,128,0,0.10969066619873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,6144,32,0,0.15706666310628256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,6144,64,0,0.11486400167147319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,6144,32,0,0.15013333161671957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,6144,4,0,0.6943626403808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,6144,8,0,0.3879893223444621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,6144,1,0,2.5443092981974282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,6144,64,0,0.10637332995732625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,8192,4,0,1.1809546947479248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,8192,64,0,0.1481013298034668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,6144,128,0,0.09908800323804219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,8192,16,0,0.37142932415008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,8192,32,0,0.24048533042271933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,8192,8,0,0.6378026803334554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,8192,2,0,2.3638505935668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,8192,128,0,0.14219733079274496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,8192,1,0,4.588656107584636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,8192,1,0,4.157813390096028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,8192,64,0,0.1402293344338735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,8192,2,0,2.123823960622152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,8192,8,0,0.6064960161844889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,8192,16,0,0.3519359827041626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,8192,4,0,1.1040960152943928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,8192,32,0,0.2246346672375997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,10240,2,0,3.762447992960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,8192,128,0,0.12966400384902954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,10240,32,0,0.33112533887227374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,10240,8,0,0.9474133650461832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,10240,16,0,0.5314720074335734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,10240,64,0,0.21665066480636597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,10240,1,0,7.648650487263997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,10240,4,0,1.7973333994547527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,10240,128,0,0.1771893302599589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,10240,8,0,0.8750133514404297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,10240,4,0,1.6370347340901692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,10240,1,0,6.263434727986653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,10240,16,0,0.48875200748443604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,10240,64,0,0.20307199160257974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,10240,2,0,3.1226879755655923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,10240,128,0,0.15875200430552164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,10240,32,0,0.2976800004641215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,12288,16,0,0.7312853336334229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,12288,32,0,0.43481600284576416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,12288,4,0,2.568650722503662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,12288,8,0,1.3223413626352947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,12288,64,0,0.2789333264033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,12288,128,0,0.21125332514444986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,12288,2,0,5.449855804443359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,12288,1,0,10.676010131835938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,12288,2,0,4.336986541748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,12288,8,0,1.177290678024292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,12288,4,0,2.2292906443277993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,12288,16,0,0.6550293366114298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,12288,32,0,0.3903040091196696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,12288,1,0,8.877311706542969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,12288,64,0,0.2587733268737793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,12288,128,0,0.19497599204381308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,1,16384,128,0,0.2813226580619812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,1,16384,64,0,0.442848006884257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,1,16384,32,0,0.6887786388397217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,1,16384,16,0,1.2012906869252522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,1,16384,8,0,2.205354690551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,1,16384,2,0,9.15449078877767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,1,16384,4,0,4.790725390116374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,1,16384,1,0,18.050132751464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,1,16384,4,0,3.8062025705973306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,1,16384,32,0,0.6133653322855631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,1,16384,16,0,1.0690826574961345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,1,16384,2,0,7.665733337402344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,1,16384,64,0,0.39341334501902264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,1,16384,8,0,1.964794635772705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,1,16384,1,0,15.297823588053385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,16,1,0,0.021210665504137676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,1,16384,128,0,0.25403199593226117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,16,2,0,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,16,8,0,0.016442666451136272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,16,4,0,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,16,32,0,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,16,16,0,0.015935999651749928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,16,128,0,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,16,64,0,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,16,2,0,0.0240639994541804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,16,1,0,0.02462933212518692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,16,4,0,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,16,8,0,0.020848001043001812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,16,32,0,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,16,64,0,0.019472000499566395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,16,16,0,0.019530666371186573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,16,128,0,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,32,2,0,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,32,1,0,0.02178666740655899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,32,4,0,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,32,32,0,0.016106666376193363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,32,16,0,0.016224000602960587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,32,8,0,0.015674666812022526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,32,64,0,0.015962666521469753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,32,128,0,0.016063999384641647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,32,1,0,0.026021334032217663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,32,2,0,0.022272000710169475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,32,4,0,0.022255999346574146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,32,8,0,0.021290667355060577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,32,16,0,0.021295999487241108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,64,1,0,0.023797333240509033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,32,64,0,0.02048533285657565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,32,128,0,0.020949333906173706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,64,2,0,0.022554665803909302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,32,32,0,0.02033599962790807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,64,4,0,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,64,8,0,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,64,16,0,0.01672533278663953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,64,32,0,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,64,1,0,0.028501334289709728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,64,128,0,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,64,2,0,0.022426667312781017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,64,64,0,0.016757333030303318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,64,4,0,0.02160000056028366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,64,32,0,0.02161066730817159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,64,16,0,0.02165333429972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,64,8,0,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,64,64,0,0.020474666108687718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,64,128,0,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,128,1,0,0.03496533383925756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,128,4,0,0.01941866676012675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,128,2,0,0.02248000105222066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,128,16,0,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,128,8,0,0.018805333723624546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,128,32,0,0.018426666657129925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,128,64,0,0.017749333133300144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,128,128,0,0.017456000049908955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,128,1,0,0.04878933231035868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,128,2,0,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,128,4,0,0.03047466774781545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,128,8,0,0.021568000316619873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,128,16,0,0.021914665897687275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,128,32,0,0.02120000123977661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,256,4,0,0.02369600037733714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,128,64,0,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,128,128,0,0.020986666282018025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,256,1,0,0.05708266794681549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,256,2,0,0.03573866685231527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,256,8,0,0.020901332298914593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,256,16,0,0.02056533346573512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,256,64,0,0.0205226664741834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,256,32,0,0.020645332833131153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,256,128,0,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,256,4,0,0.029194665451844532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,256,1,0,0.08590933680534363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,256,8,0,0.028512001037597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,256,2,0,0.05204799771308899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,256,16,0,0.02380799998839696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,256,32,0,0.023872000475724537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,256,64,0,0.023232000569502514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,256,128,0,0.02346666653951009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,512,1,0,0.10668266812960307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,512,4,0,0.04261333247025808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,512,2,0,0.062496001521746315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,512,32,0,0.02465066562096278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,512,16,0,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,512,8,0,0.026586666703224182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,512,64,0,0.023530667026837666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,512,128,0,0.023333333432674408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,512,1,0,0.16914665699005127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,512,2,0,0.09401599566141765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,512,8,0,0.03236799935499827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,512,4,0,0.05894933144251505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,512,64,0,0.026821332673231762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,512,16,0,0.028714666763941448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,512,32,0,0.02773866554101308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,1024,1,0,0.2523413300514221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,512,128,0,0.027749332288901012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,1024,2,0,0.1387999951839447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,1024,4,0,0.08202133576075236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,1024,128,0,0.0325546662012736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,1024,8,0,0.0543093333641688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,1024,64,0,0.033189333975315094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,1024,1,0,0.36559998989105225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,1024,16,0,0.034272000193595886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,1024,32,0,0.03330666571855545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,1024,2,0,0.19603200753529867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,1024,4,0,0.11117333173751831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,1024,8,0,0.07121600210666656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,1536,1,0,0.45181334018707275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,1024,32,0,0.03545066714286804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,1024,64,0,0.03500800083080927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,1024,16,0,0.04126933217048645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,1024,128,0,0.03378133227427801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,1536,2,0,0.24596800406773886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,1536,8,0,0.08597866694132487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,1536,4,0,0.13954133788744608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,1536,16,0,0.05912533402442932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,1536,1,0,0.602293332417806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,1536,2,0,0.3184266686439514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,1536,64,0,0.04057066639264425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,1536,128,0,0.0390079990029335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,1536,32,0,0.04141866664091746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,1536,4,0,0.17820799350738525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,1536,8,0,0.10551466544469197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,1536,32,0,0.04457599918047587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,1536,16,0,0.0687253326177597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,1536,128,0,0.04222933451334635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,1536,64,0,0.04161600023508072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,2048,8,0,0.12474667032559712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,2048,1,0,0.7184906800587972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,2048,2,0,0.3822346528371175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,2048,4,0,0.21303999423980713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,2048,32,0,0.05266133447488149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,2048,16,0,0.08137066662311554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,2048,64,0,0.05014933149019877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,2048,128,0,0.047695999344189964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,2048,1,0,0.894864002863566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,2048,2,0,0.46745065848032635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,2048,4,0,0.25447465976079303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,2048,8,0,0.14641599853833517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,2048,32,0,0.056645333766937256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,2048,16,0,0.09243200222651164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,2048,64,0,0.05035733183224996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,2048,128,0,0.048245335618654885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,3072,1,0,1.4337600072224934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,3072,2,0,0.7406986554463705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,3072,4,0,0.39929600556691486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,3072,8,0,0.22869332631429037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,3072,32,0,0.09236799677213033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,3072,16,0,0.13852266470591226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,3072,64,0,0.06725333134333293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,3072,128,0,0.0652106652657191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,3072,1,0,1.6271840731302898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,3072,2,0,0.83350936571757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,3072,64,0,0.07003200054168701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,3072,8,0,0.246671994527181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,3072,16,0,0.14772799611091614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,3072,32,0,0.09815999865531921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,3072,4,0,0.44701866308848065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,4096,4,0,0.6437386671702067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,3072,128,0,0.06651199857393901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,4096,16,0,0.21107200781504312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,4096,64,0,0.08760000268618266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,4096,8,0,0.35485867659250897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,4096,32,0,0.1383519967397054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,4096,1,0,2.461946646372477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,4096,2,0,1.247440020243327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,4096,128,0,0.08191466828187306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,4096,2,0,1.301909367243449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,4096,1,0,2.5394506454467773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,4096,16,0,0.21528534094492593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,4096,4,0,0.6890347003936768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,4096,8,0,0.37139733632405597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,4096,64,0,0.08819199601809184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,4096,128,0,0.07914666831493378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,4096,32,0,0.1416373352209727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,6144,8,0,0.7064533233642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,6144,1,0,5.753664016723633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,6144,64,0,0.16171200076738992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,6144,32,0,0.2421333392461141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,6144,16,0,0.39401066303253174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,6144,128,0,0.11982400218645732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,6144,4,0,1.3422400156656902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,6144,2,0,2.678874651590983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,6144,1,0,5.050367991129558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,6144,4,0,1.3109386761983235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,6144,8,0,0.6907306512196859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,6144,32,0,0.23360532522201538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,6144,16,0,0.38591468334198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,6144,2,0,2.5385120709737143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,6144,64,0,0.15796266992886862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,6144,128,0,0.1181173324584961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,8192,8,0,1.1922026475270588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,8192,64,0,0.24798399209976196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,8192,16,0,0.6363946596781412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,8192,32,0,0.37507200241088867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,8192,2,0,4.761338551839192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,8192,128,0,0.15686933199564615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,8192,4,0,2.432783921559652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,8192,1,0,9.578688303629557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,8192,1,0,8.626415888468424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,8192,32,0,0.3538666566212972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,8192,8,0,1.1159839630126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,8192,4,0,2.131930669148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,8192,64,0,0.22869867086410522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,8192,16,0,0.6096160014470419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,8192,2,0,4.255392074584961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,8192,128,0,0.1511733333269755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,10240,2,0,7.512975692749023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,10240,32,0,0.5374826590220133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,10240,64,0,0.3416479825973511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,10240,16,0,0.953285296758016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,10240,128,0,0.22518932819366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,10240,8,0,1.766965389251709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,10240,4,0,3.8652852376302085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,10240,1,0,14.913883209228516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,10240,128,0,0.20885332425435385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,10240,16,0,0.8811786969502767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,10240,32,0,0.49827734629313153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,10240,64,0,0.30828267335891724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,10240,8,0,1.6328852971394856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,10240,1,0,12.568373362223307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,10240,4,0,3.1745227177937827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,10240,2,0,6.20846430460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,12288,32,0,0.7343466281890869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,12288,64,0,0.44438934326171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,12288,128,0,0.2932586669921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,12288,16,0,1.3455467224121094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,12288,8,0,2.5773706436157227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,12288,4,0,5.420949300130208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,12288,2,0,10.654591878255209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,12288,1,0,23.26258595784505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,12288,32,0,0.6679360071818033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,12288,64,0,0.4100480079650879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,12288,16,0,1.2125600179036458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,12288,8,0,2.297248045603434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,12288,128,0,0.26450665791829425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,12288,4,0,4.442517280578613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,12288,2,0,8.77837880452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,12288,1,0,17.467530568440754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,2,16384,16,0,2.447546641031901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,2,16384,128,0,0.4527466694513957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,2,16384,64,0,0.7040747006734213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,2,16384,32,0,1.1976426442464192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,2,16384,8,0,4.8090667724609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,2,16384,4,0,9.284405390421549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,2,16384,8,0,3.909989356994629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,2,16384,4,0,7.664239883422852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,2,16384,16,0,1.9593706130981445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,2,16384,32,0,1.06168532371521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,2,16384,2,0,18.47425587972005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,2,16384,64,0,0.6153173446655273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,2,16384,2,0,15.068703969319662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,2,16384,128,0,0.40405865510304767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,16,1,0,0.028101332485675812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,16,2,0,0.022074667116006214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,16,16,0,0.01609066625436147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,16,8,0,0.016415999581416447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,2,16384,1,0,41.841270446777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,16,4,0,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,16,32,0,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,16,64,0,0.015637333194414776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,16,128,0,0.016074666132529575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,2,16384,1,0,37.238606770833336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,16,4,0,0.022863999009132385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,16,2,0,0.025829332570234936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,16,1,0,0.03196266790231069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,16,8,0,0.022682666778564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,16,16,0,0.02067733307679494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,16,64,0,0.020799999435742695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,16,32,0,0.020256000260512035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,16,128,0,0.021882665654023487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,32,2,0,0.021925332645575207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,32,8,0,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,32,1,0,0.02923733244339625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,32,4,0,0.018191999445358913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,32,16,0,0.0161013330022494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,32,32,0,0.01637866720557213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,32,64,0,0.01562133307258288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,32,2,0,0.026122666895389557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,32,1,0,0.03472000112136205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,32,128,0,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,32,4,0,0.022005334496498108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,32,32,0,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,32,16,0,0.02089066555102666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,32,8,0,1.3244799772898357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,32,64,0,0.02041600023706754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,32,128,0,0.02075733368595441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,64,1,0,0.037733333806196846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,64,4,0,0.019498666127522785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,64,2,0,0.02369600037733714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,64,8,0,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,64,16,0,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,64,64,0,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,64,128,0,0.01659199967980385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,64,32,0,0.01634666696190834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,64,1,0,0.0543146679798762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,64,2,0,0.028944000601768494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,64,4,0,0.023157333334287006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,64,8,0,0.022015998760859173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,64,16,0,0.021882665654023487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,64,64,0,0.020645332833131153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,64,32,0,0.02120000123977661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,128,1,0,0.05702400207519531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,64,128,0,0.020645332833131153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,128,8,0,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,128,4,0,0.02292799949645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,128,2,0,0.03517866631348928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,128,16,0,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,128,32,0,0.0182239996890227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,128,128,0,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,128,64,0,0.01806933308641116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,128,1,0,0.08212799827257793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,128,2,0,0.05248000224431356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,128,32,0,0.022298666338125866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,128,16,0,0.022389332453409832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,128,4,0,0.026506667335828144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,128,8,0,0.02316266546646754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,128,64,0,0.021290667355060577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,128,128,0,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,256,1,0,0.10087999701499939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,256,2,0,0.05827199916044871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,256,4,0,0.03629333277543386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,256,8,0,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,256,16,0,0.021669333179791767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,256,64,0,0.02089066555102666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,256,32,0,0.021498667697111767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,256,128,0,0.019808000574509304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,256,2,0,0.08628267049789429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,256,4,0,0.051882664362589516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,256,1,0,0.15574933091799417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,256,8,0,0.02867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,256,16,0,0.02462399999300639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,256,32,0,0.024101334313551586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,256,128,0,0.02258133391539256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,256,64,0,0.023733332753181458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,512,2,0,0.10591466228167216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,512,1,0,0.19195199012756348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,512,4,0,0.06389866769313812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,512,8,0,0.042912001411120095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,512,16,0,0.027722666660944622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,512,32,0,0.025839999318122864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,512,64,0,0.024351999163627625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,512,1,0,0.3109653393427531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,512,128,0,0.024351999163627625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,512,2,0,0.16929600636164346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,512,4,0,0.0941386620203654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,512,8,0,0.0589279979467392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,512,16,0,0.033861334125200905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,512,64,0,0.02757866680622101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,512,32,0,0.02864533414443334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,1024,1,0,0.4685973326365153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,512,128,0,0.02738133321205775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,1024,4,0,0.14174933234850565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,1024,2,0,0.24945600827534994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,1024,16,0,0.055717334151268005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,1024,32,0,0.036373332142829895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,1024,8,0,0.08411199847857158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,1024,64,0,0.03378133227427801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,1024,1,0,0.6901386578877767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,1024,128,0,0.032645332316557564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,1024,2,0,0.36420265833536786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,1024,8,0,0.11195733149846394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,1024,16,0,0.07209066549936931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,1024,4,0,0.1962826649347941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,1024,32,0,0.04407466451327006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,1024,64,0,0.0354720006386439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,1024,128,0,0.035018667578697205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,1536,2,0,0.46051732699076336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,1536,1,0,0.9040746688842773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,1536,32,0,0.062650665640831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,1536,8,0,0.14500799775123596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,1536,64,0,0.043925335009892784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,1536,4,0,0.24836800495783487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,1536,16,0,0.08713600039482117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,1536,128,0,0.04190400242805481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,1536,1,0,1.171130657196045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,1536,16,0,0.1081813375155131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,1536,8,0,0.17942400773366293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,1536,4,0,0.3227840065956116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,1536,2,0,0.6018879810969034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,1536,32,0,0.07127999762694041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,1536,64,0,0.048010667165120445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,1536,128,0,0.043663998444875084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,2048,1,0,1.4353334108988445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,2048,2,0,0.7211360136667887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,2048,4,0,0.38709867000579834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,2048,8,0,0.2182240088780721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,2048,32,0,0.08415466547012329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,2048,16,0,0.12827733159065247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,2048,128,0,0.05162133276462555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,2048,64,0,0.05572799841562907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,2048,1,0,1.749658743540446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,2048,2,0,0.8992213408152262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,2048,4,0,0.47062933444976807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,2048,8,0,0.2590399980545044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,2048,64,0,0.06132266422112783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,2048,32,0,0.09459732969601949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,2048,16,0,0.14708800117174783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,2048,128,0,0.052560001611709595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,3072,1,0,2.918794631958008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,3072,4,0,0.7573119799296061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,3072,2,0,1.4445279439290364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,3072,8,0,0.4055200020472209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,3072,16,0,0.23270400365193686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,3072,64,0,0.09662933150927226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,3072,32,0,0.14434132973353067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,3072,128,0,0.07381333410739899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,3072,2,0,1.627786636352539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,3072,1,0,3.198047955830892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,3072,4,0,0.8511892954508463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,3072,8,0,0.45099735260009766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,3072,16,0,0.25336533784866333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,3072,32,0,0.15243732929229736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,3072,64,0,0.1030613382657369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,3072,128,0,0.07502399881680806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,4096,4,0,1.231280008951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,4096,2,0,2.458677291870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,4096,16,0,0.36323734124501544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,4096,8,0,0.6520373423894247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,4096,1,0,5.346058527628581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,4096,32,0,0.21760533253351846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,4096,64,0,0.14406399925549826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,4096,128,0,0.09353599945704143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,4096,4,0,1.3158880074818928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,4096,32,0,0.21935999393463135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,4096,16,0,0.39821334679921466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,4096,8,0,0.6896373430887858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,4096,2,0,2.547935962677002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,4096,1,0,5.038496017456055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,4096,128,0,0.09432533383369446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,4096,64,0,0.1432319978872935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,6144,2,0,5.673797607421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,6144,128,0,0.17138133446375528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,6144,4,0,2.649845282236735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,6144,32,0,0.4113066593805949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,6144,8,0,1.3335307439168294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,6144,64,0,0.250490665435791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,6144,16,0,0.722383975982666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,6144,1,0,11.328144073486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,6144,2,0,5.246682802836101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,6144,1,0,10.600512186686197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,6144,32,0,0.39538665612538654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,6144,16,0,0.7094133694966634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,6144,64,0,0.24389866987864176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,6144,4,0,2.587242603302002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,6144,8,0,1.324181318283081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,6144,128,0,0.16969066858291626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,4,8192,32,0,0.650325338045756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,4,8192,64,0,0.38995198408762616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,4,8192,4,0,4.596997261047363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,4,8192,16,0,1.204650640487671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,4,8192,8,0,2.296346664428711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,4,8192,2,0,9.798458735148111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,4,8192,128,0,0.2601919968922933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,4,8192,1,0,23.8209228515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,4,8192,8,0,2.1674133936564126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,4,8192,32,0,0.6245973507563273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,4,8192,64,0,0.3612639904022217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,4,8192,4,0,4.309365272521973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,4,8192,16,0,1.131930669148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,4,8192,2,0,8.420469284057617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,4,8192,128,0,0.24270933866500854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,16,1,0,0.04321066538492838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,16,2,0,0.029077333708604176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,16,4,0,0.021114667256673176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,4,8192,1,0,17.178117116292317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,16,8,0,0.018746666610240936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,16,16,0,0.016250666230916977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,16,64,0,0.01591466615597407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,16,32,0,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,16,1,0,0.04664533336957296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,16,128,0,0.021941334009170532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,16,2,0,0.032431999842325844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,16,4,0,0.026149332523345947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,16,8,0,0.022815999885400135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,16,16,0,0.021642667551835377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,16,64,0,0.02080533280968666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,16,32,0,0.02163200080394745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,32,2,0,0.029930666089057922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,16,128,0,0.020047999918460846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,32,1,0,0.04667733112970988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,32,4,0,0.022202665607134502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,32,8,0,0.018239999810854595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,32,16,0,0.01628799984852473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,32,32,0,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,32,64,0,0.016106666376193363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,32,128,0,0.01611199975013733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,32,8,0,0.024383999407291412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,32,4,0,0.0260959987839063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,32,1,0,0.06398933132489522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,32,2,0,0.058517331878344216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,32,64,0,0.025663999219735462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,32,32,0,0.020703999946514767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,32,16,0,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,32,128,0,0.021130666136741638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,64,4,0,0.02441066751877467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,64,2,0,0.03793599953254064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,64,32,0,0.016719999412695568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,64,1,0,0.05858133236567179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,64,8,0,0.019253333409627277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,64,16,0,0.0180479995906353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,64,64,0,0.01681600014368693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,64,128,0,0.016714667280515034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,64,1,0,0.09307733178138733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,64,2,0,0.05423999826113383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,64,32,0,0.021712000171343487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,64,4,0,0.028389332195123036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,64,64,0,0.021061333517233532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,64,16,0,0.021930667261282604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,64,8,0,0.023034666975339253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,64,128,0,0.020821332931518555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,128,1,0,0.10077333450317383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,128,2,0,0.059248000383377075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,128,4,0,0.03498133271932602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,128,32,0,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,128,8,0,0.022426667312781017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,128,16,0,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,128,64,0,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,128,1,0,0.15100266536076865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,128,128,0,0.01854933301607768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,128,2,0,0.08298133313655853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,128,8,0,0.027701333165168762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,128,4,0,0.04911999901135763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,128,16,0,0.02269333352645238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,128,64,0,0.02184533327817917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,128,128,0,0.021456000705560047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,256,1,0,0.18146665891011557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,128,32,0,0.021557333568731945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,256,2,0,0.10107733805974324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,256,4,0,0.05894400179386139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,256,16,0,0.023914667467276256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,256,64,0,0.021615999440352123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,256,8,0,0.03659199923276901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,256,32,0,0.022218666970729828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,256,128,0,0.021573332448800404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,256,1,0,0.2886773347854614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,256,2,0,0.1562933325767517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,256,4,0,0.08598400155703227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,256,8,0,0.05266133447488149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,256,16,0,0.03088533381621043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,256,32,0,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,256,64,0,0.02496533344189326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,256,128,0,0.02418133368094762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,512,2,0,0.1913386583328247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,512,1,0,0.35361599922180176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,512,4,0,0.10805867115656535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,512,8,0,0.0650186687707901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,512,16,0,0.04494399825731913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,512,64,0,0.026778665681680042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,512,32,0,0.02827200045188268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,512,128,0,0.0249439999461174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,512,8,0,0.09683199723561604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,512,2,0,0.3126506606737773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,512,4,0,0.16756800810496011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,512,1,0,0.594373345375061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,512,16,0,0.06187733511130015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,512,32,0,0.03692800054947535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,512,64,0,0.029264000554879505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,512,128,0,0.028970666229724884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,1024,2,0,0.4756053288777669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,1024,4,0,0.25457600752512616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,1024,1,0,0.9109600385030111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,1024,8,0,0.14421866337458292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,1024,16,0,0.08756800492604573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,1024,32,0,0.06011199951171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,1024,64,0,0.041135999063650765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,1024,128,0,0.035360001027584076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,1024,8,0,0.19799999396006265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,1024,2,0,0.6904906431833903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,1024,32,0,0.07412266731262207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,1024,1,0,1.353040059407552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,1024,16,0,0.1151093343893687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,1024,4,0,0.3651520013809204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,1024,64,0,0.048895999789237976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,1536,1,0,1.7286292711893718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,1536,4,0,0.47009066740671795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,1024,128,0,0.038975998759269714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,1536,2,0,0.8968746662139893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,1536,16,0,0.14993066589037576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,1536,8,0,0.25332266092300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,1536,32,0,0.09380799531936646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,1536,128,0,0.053599998354911804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,1536,64,0,0.0679253339767456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,1536,1,0,2.319525400797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,1536,4,0,1.0153226852416992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,1536,2,0,1.173967997233073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,1536,8,0,0.3251253366470337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,1536,16,0,0.18207999070485434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,1536,32,0,0.11311999956766765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,1536,64,0,0.07502933343251546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,2048,1,0,3.150341351826986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,2048,64,0,0.09244799613952637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,1536,128,0,0.055248002211252846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,2048,8,0,0.39396798610687256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,2048,4,0,0.7322666645050049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,2048,2,0,1.4473919868469238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,2048,32,0,0.13740266362826029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,2048,16,0,0.22179200251897177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,2048,128,0,0.06380266447861989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,2048,1,0,3.4755093256632485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,2048,2,0,1.7586453755696614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,2048,16,0,0.262554665406545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,2048,128,0,0.06866666674613953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,2048,4,0,0.9024266401926676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,2048,8,0,0.4755946795145671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,2048,32,0,0.15461867054303488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,2048,64,0,0.10101866722106934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,3072,1,0,6.149066925048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,3072,2,0,2.947397232055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,3072,8,0,0.7801600297292074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,3072,4,0,1.5172959963480632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,3072,16,0,0.41597867012023926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,3072,128,0,0.10800000031789143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,3072,32,0,0.24272000789642334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,3072,64,0,0.15638400117556253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,3072,4,0,1.6454720497131348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,3072,1,0,6.38262939453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,3072,2,0,3.2267414728800454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,3072,8,0,0.8553333282470703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,3072,32,0,0.25814932584762573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,3072,16,0,0.4567893346150716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,3072,128,0,0.11133333047231038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,3072,64,0,0.16165866454442343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,8,4096,2,0,5.314474741617839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,8,4096,8,0,1.2753760019938152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,8,4096,16,0,0.6703680356343588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,8,4096,128,0,0.1588479975859324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,8,4096,32,0,0.3772159814834595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,8,4096,64,0,0.23145065704981485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,8,4096,4,0,2.455728054046631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,8,4096,1,0,12.309076944986979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,8,4096,128,0,0.15946132938067117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,8,4096,1,0,10.12991460164388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,8,4096,16,0,0.7023466428120931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,8,4096,32,0,0.3914560079574585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,8,4096,64,0,0.24341867367426553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,8,4096,2,0,5.225935935974121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,8,4096,8,0,1.3337920506795247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,8,4096,4,0,2.588197390238444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,16,1,0,0.07367466886838277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,16,2,0,0.041893333196640015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,16,4,0,0.029845332105954487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,16,128,0,0.015909332782030106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,16,8,0,0.02202133337656657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,16,32,0,0.01643199970324834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,16,16,0,0.020714666694402695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,16,2,0,0.04539200166861216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,16,1,0,0.0857919951279958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,16,64,0,0.016789333273967106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,16,4,0,0.03146133323510488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,16,64,0,0.020442667106787365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,16,128,0,0.020741333564122517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,16,8,0,0.02607999990383784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,16,16,0,0.021941334009170532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,16,32,0,0.021669333179791767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,32,1,0,0.07835199932257335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,32,2,0,0.048351998130480446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,32,4,0,0.029050665597120922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,32,8,0,0.021082667013009388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,32,16,0,0.01833600054184596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,32,32,0,0.016677333662907284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,32,2,0,0.06381333371003468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,32,64,0,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,32,128,0,0.016202667107184727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,32,4,0,0.033813332517941795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,32,1,0,0.11454932888348897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,32,16,0,0.022698665658632915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,32,8,0,0.026000000536441803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,32,64,0,0.021370666722456615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,32,32,0,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,64,1,0,0.09985599915186565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,64,2,0,0.058362667759259544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,32,128,0,0.021551998953024547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,64,4,0,0.03810133288304011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,64,32,0,0.01794133335351944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,64,16,0,0.019765333582957584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,64,8,0,0.02425066630045573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,64,1,0,0.1695680022239685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,64,64,0,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,64,128,0,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,64,2,0,0.09417066971460979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,64,8,0,0.02870933214823405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,64,4,0,0.05439466734727224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,64,64,0,0.021370666722456615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,64,16,0,0.02325333406527837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,64,32,0,0.022005334496498108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,64,128,0,0.020986666282018025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,128,1,0,0.1821920077006022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,128,4,0,0.058133333921432495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,128,2,0,0.10113599896430969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,128,8,0,0.03603733330965042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,128,16,0,0.023743999501069386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,128,32,0,0.020303999384244282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,128,2,0,0.15124799807866415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,128,1,0,0.27879999081293744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,128,128,0,0.018197332819302876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,128,64,0,0.01934933289885521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,128,16,0,0.030117332935333252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,128,4,0,0.08365866541862488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,128,8,0,0.050026665131251015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,128,32,0,0.023152001202106476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,128,64,0,0.022053333620230358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,256,4,0,0.10281599561373393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,128,128,0,0.02204799900452296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,256,1,0,0.3382986783981323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,256,2,0,0.18078400691350302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,256,16,0,0.039247999588648476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,256,64,0,0.02258666604757309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,256,8,0,0.060789331793785095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,256,128,0,0.02214933435122172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,256,32,0,0.026005332668622334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,256,1,0,0.5565333366394043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,256,2,0,0.2895039916038513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,256,4,0,0.157642662525177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,256,16,0,0.05470933516820272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,256,8,0,0.08846933643023173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,256,64,0,0.025594666600227356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,256,128,0,0.02441066751877467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,256,32,0,0.03410666684309641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,512,8,0,0.11095999677975972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,512,2,0,0.3593706687291463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,512,4,0,0.19484800100326538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,512,1,0,0.686405340830485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,512,32,0,0.04868266483147939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,512,16,0,0.07035733262697856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,512,128,0,0.028368001182874043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,512,64,0,0.032799998919169106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,512,1,0,1.1753973166147869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,512,16,0,0.09858666857083638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,512,4,0,0.3147146701812744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,512,2,0,0.5980000098546346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,512,8,0,0.17192532618840536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,512,32,0,0.06408533453941345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,512,64,0,0.04312000175317129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,512,128,0,0.032399999598662056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,1024,1,0,1.8054347038269043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,1024,4,0,0.4797919988632202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,1024,2,0,0.9469013214111328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,1024,8,0,0.262501339117686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,1024,16,0,0.15385066469510397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,1024,64,0,0.06929600238800049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,1024,32,0,0.09547199805577596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,1024,128,0,0.048938666780789696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,1024,2,0,1.3610186576843262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,1024,16,0,0.2051680088043213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,1024,4,0,0.6992586453755697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,1024,1,0,2.6821492513020835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,1024,8,0,0.3917173147201538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,1024,32,0,0.12124266227086385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,1024,64,0,0.08004799981911977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,1024,128,0,0.05909333129723867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,1536,16,0,0.26665065685908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,1536,32,0,0.16250666975975037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,1536,8,0,0.4787360032399495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,1536,4,0,0.8907306989034017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,1536,64,0,0.10456533233324687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,1536,2,0,1.8087414105733235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,1536,1,0,3.66105588277181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,1536,128,0,0.07771733403205872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,1536,2,0,2.329514662424723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,1536,16,0,0.33322133620580036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,1536,1,0,4.6554826100667315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,1536,8,0,0.619322657585144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,1536,4,0,1.1856533686319988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,1536,128,0,0.0844053328037262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,1536,32,0,0.19187732537587485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,1536,64,0,0.12939199805259705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,16,2048,2,0,2.8826773961385093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,16,2048,1,0,6.1975148518880205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,16,2048,4,0,1.453221321105957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,16,2048,128,0,0.10757333040237427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,16,2048,32,0,0.2377600073814392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,16,2048,16,0,0.40720001856486004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,16,2048,64,0,0.1504853367805481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,16,2048,8,0,0.7421226501464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,16,2048,8,0,0.9234240055084229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,16,2048,1,0,6.981546401977539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,16,2048,2,0,3.510666529337565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,16,2048,32,0,0.2734453280766805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,16,2048,4,0,1.7864747047424316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,16,2048,16,0,0.48786131540934247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,16,2048,64,0,0.16370133558909097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,16,2048,128,0,0.11178132891654968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,16,1,0,0.1369599997997284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,16,2,0,0.07434133191903432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,16,8,0,0.028517333169778187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,16,4,0,0.042165334026018776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,16,32,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,16,16,0,0.020661332954963047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,16,128,0,0.0162773331006368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,16,64,0,0.01785600061217944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,16,1,0,0.15569600462913513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,16,2,0,0.08502933382987976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,16,8,0,0.03250666707754135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,16,4,0,0.04730666677157084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,16,32,0,0.02252800017595291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,16,16,0,0.025637333591779072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,16,64,0,0.021488000949223835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,32,1,0,0.14010133345921835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,32,2,0,0.0768746683994929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,16,128,0,0.02081599955757459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,32,4,0,0.048138668139775596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,32,16,0,0.02189333240191142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,32,8,0,0.03356799980004629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,32,32,0,0.01826133330663045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,32,128,0,0.01651200031240781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,32,1,0,0.20954134066899618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,32,64,0,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,32,2,0,0.11126933495203654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,32,4,0,0.06471999982992808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,32,8,0,0.03613866617282232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,32,16,0,0.026144000391165417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,32,32,0,0.0223786657055219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,32,128,0,0.021551998953024547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,32,64,0,0.021674667795499165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,64,1,0,0.1820746660232544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,64,2,0,0.10187733173370361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,64,4,0,0.06043200194835663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,64,8,0,0.03862933317820231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,64,16,0,0.025402667621771496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,64,32,0,0.01995733380317688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,64,64,0,0.018144000321626663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,64,128,0,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,64,8,0,0.055071999629338585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,64,4,0,0.09656533598899841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,64,1,0,0.317029337088267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,64,64,0,0.0227360005180041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,64,16,0,0.033088001112143196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,64,2,0,0.17053866386413574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,64,128,0,0.02199999988079071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,64,32,0,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,128,1,0,0.34727466106414795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,128,2,0,0.1832853356997172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,128,8,0,0.06012799839178721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,128,16,0,0.03903999924659729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,128,4,0,0.10404800375302632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,128,64,0,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,128,128,0,0.02035733312368393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,128,32,0,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,128,1,0,0.5356106758117676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,128,2,0,0.28069865703582764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,128,4,0,0.15291733543078104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,128,8,0,0.08581866820653279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,128,32,0,0.033887999753157295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,128,64,0,0.024432001014550526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,128,16,0,0.05151999990145365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,128,128,0,0.023002666731675465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,256,1,0,0.6578880151112875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,256,2,0,0.33896533648173016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,256,4,0,0.18545599778493246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,256,16,0,0.06839466591676076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,256,8,0,0.1071626643339793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,256,32,0,0.043231998880704246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,256,128,0,0.02476266771554947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,256,64,0,0.029872000217437744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,256,2,0,0.5561600128809611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,256,1,0,1.090336004892985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,256,4,0,0.29315733909606934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,256,8,0,0.16050133109092712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,256,16,0,0.09134399890899658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,256,32,0,0.057349334160486855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,256,64,0,0.03855466594298681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,512,1,0,1.3465226491292317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,512,2,0,0.6877600351969401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,256,128,0,0.02847466617822647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,512,4,0,0.3636000156402588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,512,8,0,0.19974400599797568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,512,64,0,0.057445332407951355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,512,16,0,0.11877866586049397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,512,128,0,0.04465066889921824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,512,32,0,0.0786186655362447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,512,1,0,2.3326826095581055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,512,8,0,0.3184746702512105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,512,4,0,0.6023786862691244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,512,2,0,1.1765813032786052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,512,16,0,0.17762132485707602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,512,32,0,0.10447999835014343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,512,64,0,0.06922666728496552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,512,128,0,0.04857600231965383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,32,1024,1,0,3.725104014078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,32,1024,2,0,1.8917279243469238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,32,1024,4,0,0.9477120240529379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,32,1024,16,0,0.27691733837127686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,32,1024,8,0,0.5022933483123779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,32,1024,64,0,0.11039466659228007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,32,1024,32,0,0.1667893330256144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,32,1024,128,0,0.08160533507664998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,32,1024,8,0,0.7158079942067465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,32,1024,4,0,1.3887306849161785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,32,1024,2,0,2.7365118662516275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,32,1024,32,0,0.21635200579961142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,32,1024,16,0,0.38233065605163574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,32,1024,64,0,0.13168533643086752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,32,1024,1,0,5.436906814575195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,64,16,1,0,0.2659200032552083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,32,1024,128,0,0.09031466643015544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,16,4,0,0.07355199754238129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,16,2,0,0.13423466682434082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,16,8,0,0.04206933577855428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,16,16,0,0.02810666710138321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,16,32,0,0.020367999871571858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,16,64,0,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,64,16,128,0,0.016528000434239704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,64,16,1,0,0.300053338209788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,16,2,0,0.15559466679890951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,16,4,0,0.08611200253168742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,16,8,0,0.048245335618654885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,16,16,0,0.03134933362404505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,16,64,0,0.02207999924818675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,64,16,128,0,0.021930667261282604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,16,32,0,0.025781333446502686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,32,2,0,0.13948266704877219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,32,8,0,0.04710400104522705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,32,4,0,0.0759093314409256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,64,32,1,0,0.2767680088678996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,32,16,0,0.02899733434120814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,32,64,0,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,32,32,0,0.021551998953024547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,64,32,128,0,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,64,32,1,0,0.39645334084828693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,32,4,0,0.11019200086593628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,32,2,0,0.2092640002568563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,32,8,0,0.065461332599322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,32,64,0,0.022842665513356526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,32,32,0,0.026650667190551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,64,64,1,0,0.34701331456502277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,64,2,0,0.18481600284576416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,64,4,0,0.10613333185513814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,64,32,128,0,0.022287999590237934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,32,16,0,0.03667200108369192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,64,8,0,0.06198933223883311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,64,16,0,0.04081066697835922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,64,64,0,0.021456000705560047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,64,32,0,0.026144000391165417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,64,64,1,0,0.6149653196334839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,64,64,128,0,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,64,2,0,0.3189599911371867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,64,4,0,0.17065600554148355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,64,16,0,0.056128000219662987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,64,64,0,0.024138666689395905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,64,32,0,0.035455999275048576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,64,8,0,0.09373333056767781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,64,64,128,0,0.022991999983787537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,64,128,1,0,0.6596213181813558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,128,2,0,0.34488534927368164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,128,16,0,0.06613866488138835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,128,4,0,0.18799465894699097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,128,8,0,0.1074666678905487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,128,32,0,0.042362665136655174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,128,64,0,0.029552000264326733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,64,128,128,0,0.023973333338896435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,128,2,0,0.5383946498235067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,128,16,0,0.08815466364224751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,128,8,0,0.15599466363588968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,128,4,0,0.2822879950205485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,64,128,1,0,1.0508480072021484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,128,32,0,0.05428266525268555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,128,64,0,0.037962667644023895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,64,128,128,0,0.026789332429567974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,256,2,0,0.6602559884389242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,256,4,0,0.3482133150100708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,64,256,1,0,1.2956106662750244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,256,8,0,0.19055465857187906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,256,16,0,0.11615999539693196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,256,64,0,0.05287999908129374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,64,256,128,0,0.03957866628964742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,256,32,0,0.07326933244864146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,64,256,1,0,2.176469326019287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,256,4,0,0.5621653397878011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,256,2,0,1.0999200344085693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,256,16,0,0.16699200868606567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,256,8,0,0.2969226638476054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,256,32,0,0.09657067060470581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,256,64,0,0.06364800035953522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,64,256,128,0,0.04554133117198944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,64,512,1,0,2.7685813903808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,64,512,2,0,1.3663466771443684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,64,512,4,0,0.7009759744008383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,64,512,64,0,0.09215999643007915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,64,512,8,0,0.3772053321202596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,64,512,32,0,0.13262400031089783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,64,512,16,0,0.21463465690612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,64,512,128,0,0.07162133355935414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,64,512,8,0,0.6185173193613688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,64,512,2,0,2.3825653394063315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,64,512,4,0,1.2000693480173747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,64,512,1,0,4.771738688151042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,64,512,32,0,0.1881386637687683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,64,512,16,0,0.331770658493042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,64,512,64,0,0.11454932888348897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,128,16,1,0,0.5235199928283691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,64,512,128,0,0.08040000001589458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,16,2,0,0.26264532407124835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,16,8,0,0.0740479975938797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,16,16,0,0.041877334316571556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,16,4,0,0.13641066352526346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,16,32,0,0.027471999327341717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,16,64,0,0.02011200040578842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,128,16,128,0,0.017674667139848072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,16,2,0,0.29870400826136273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,128,16,1,0,0.5816533168156942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,16,8,0,0.08558400472005208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,16,4,0,0.15523200233777365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,16,16,0,0.048714667558670044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,16,32,0,0.032111999889214836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,16,64,0,0.025637333591779072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,128,16,128,0,0.022661333282788593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,128,32,1,0,0.5471733411153158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,32,2,0,0.27717334032058716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,32,8,0,0.0787360022465388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,32,4,0,0.1427786648273468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,32,16,0,0.04836266736189524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,128,32,128,0,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,32,32,0,0.030117332935333252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,32,64,0,0.022287999590237934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,128,32,1,0,0.7697599728902181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,32,4,0,0.20934933423995972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,32,8,0,0.1146506667137146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,32,2,0,0.3968799908955892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,32,16,0,0.0680320014556249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,32,32,0,0.04002666721741358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,32,64,0,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,128,64,1,0,0.6751626332600912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,128,32,128,0,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,64,4,0,0.19188799460728964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,64,2,0,0.35259731610616046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,64,16,0,0.06676266590754192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,64,8,0,0.10825066765149434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,64,32,0,0.0439573327700297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,128,64,128,0,0.023770667612552643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,64,64,0,0.030623999734719593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,128,64,1,0,1.2055786450703938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,64,4,0,0.31968533992767334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,64,8,0,0.17468800147374472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,64,2,0,0.6155999898910522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,64,32,0,0.05923733115196228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,64,64,0,0.04032533367474874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,64,16,0,0.09660800298055013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,128,64,128,0,0.02808533360560735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,128,4,0,0.35393067200978595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,128,128,1,0,1.3006239732106526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,128,2,0,0.6663893461227417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,128,8,0,0.1924053430557251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,128,32,0,0.0754613329966863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,128,64,0,0.05279466509819031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,128,16,0,0.11539733409881592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,128,128,128,0,0.04118400067090988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,128,4,0,0.5455306768417358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,128,128,1,0,2.1158560117085776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,128,2,0,1.060757319132487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,128,8,0,0.28943467140197754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,128,32,0,0.09387200077374776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,128,16,0,0.16296533743540445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,128,128,128,0,0.04474666714668274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,128,64,0,0.060421332716941833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,128,256,8,0,0.3588693141937256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,128,256,16,0,0.20586133003234863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,128,256,2,0,1.2972533702850342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,128,256,4,0,0.6747999986012777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,128,256,1,0,2.564341386159261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,128,256,32,0,0.1273973286151886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,128,256,64,0,0.08829866846402486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,128,256,128,0,0.06788800160090129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,128,256,1,0,4.44434134165446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,128,256,4,0,1.1234827041625977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,128,256,2,0,2.2284107208251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,128,256,16,0,0.32154667377471924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,128,256,8,0,0.5815093517303467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,128,256,32,0,0.17677332957585654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,128,256,64,0,0.10842667023340861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,128,256,128,0,0.07489599784215291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,256,16,2,0,0.5213173230489095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,256,16,16,0,0.07044266661008199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,256,16,8,0,0.1354986627896627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,256,16,4,0,0.2653333346048991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,256,16,32,0,0.04271999994913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,256,16,1,0,1.0687626997629802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,256,16,64,0,0.02863466739654541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,256,16,128,0,0.021759999295075733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,256,16,1,0,1.1371359825134277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,256,16,8,0,0.15615999698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,256,16,4,0,0.2980053424835205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,256,16,32,0,0.05422399938106537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,256,16,64,0,0.03312533348798752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,256,16,128,0,0.026709333062171936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,256,16,2,0,0.5768053531646729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,256,16,16,0,0.0876639982064565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,256,32,1,0,1.1037866274515789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,256,32,4,0,0.2846026619275411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,256,32,2,0,0.5552746852238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,256,32,8,0,0.14749866724014282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,256,32,32,0,0.05388799806435903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,256,32,64,0,0.03523733218510946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,256,32,128,0,0.024714666108290356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,256,32,16,0,0.08165333171685536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,256,32,4,0,0.399397333463033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,256,32,64,0,0.04651199777921041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,256,32,2,0,0.7750133673350016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,256,32,32,0,0.07149866720040639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,256,32,1,0,1.5250719388326008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,256,32,8,0,0.21437867482503256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,256,32,16,0,0.11638933420181274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,256,32,128,0,0.03011200080315272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,256,64,2,0,0.6844533284505209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,256,64,8,0,0.19655466079711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,256,64,1,0,1.3361493746439617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,256,64,32,0,0.0745600014925003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,256,64,4,0,0.3655093510945638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,256,64,128,0,0.04049066702524821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,256,64,16,0,0.1253866652647654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,256,64,64,0,0.05362666646639506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,256,64,1,0,2.4129014015197754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,256,64,2,0,1.218949317932129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,256,64,32,0,0.10314133763313293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,256,64,8,0,0.3254186709721883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,256,64,4,0,0.624725341796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,256,64,128,0,0.047168001532554626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,256,64,16,0,0.17864000797271729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,256,64,64,0,0.06549866497516632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,128,256,128,1,0,2.6308959325154624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,64,256,128,2,0,1.3137493133544922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,32,256,128,4,0,0.6847253640492758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,2,256,128,64,0,0.08914132912953694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,4,256,128,32,0,0.12973866860071817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,16,256,128,8,0,0.3680586814880371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,1,256,128,128,0,0.06784533460934956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,float16,8,256,128,16,0,0.20811732610066733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,64,256,128,2,0,2.151408036549886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,128,256,128,1,0,4.297274589538574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,16,256,128,8,0,0.5662986834843954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,1,256,128,128,0,0.07113066812356313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,4,256,128,32,0,0.1734666625658671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,32,256,128,4,0,1.0896586577097576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,2,256,128,64,0,0.10577600200970967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_context,default,float16,fp8,8,256,128,16,0,0.3038133382797241
