framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,1,0.02197866638501485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,1,0.027600000301996868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,1,0.02606400102376938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,1,0.02568000058333079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,1,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,1,0.021930667261282604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,1,0.02555199960867564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,1,0.025450666745503742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,1,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,1,0.025733334322770435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,1,0.02629333237806956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,1,0.024746666351954143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,1,0.025077333052953083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,1,0.025045332809289295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,1,0.02571733295917511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,1,0.024698667228221893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,3,0.021712000171343487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,3,0.02606400102376938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,3,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,3,0.026730666557947796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,3,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,3,0.02603200078010559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,3,0.02534399926662445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,3,0.02641066660483678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,3,0.025173333783944447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,3,0.021850667893886566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,3,0.026496000587940216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,3,0.024901332954565685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,3,0.025807999074459076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,3,0.025360000630219776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,3,0.024735999604066212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,7,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,3,0.025519999365011852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,7,0.02712533374627431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,7,0.026554666459560394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,7,0.025594666600227356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,7,0.026176000634829204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,7,0.02603200078010559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,7,0.02606400102376938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,7,0.02609066665172577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,7,0.022069332500298817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,7,0.02649066597223282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,7,0.025610665480295818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,7,0.025861332813898723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,7,0.025279998779296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,7,0.025061334172884624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,15,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,7,0.025744001070658367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,7,0.02628266563018163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,15,0.02693866689999898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,15,0.02608533451954524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,15,0.02644266684850057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,15,0.025642665723959606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,15,0.026026666164398193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,15,0.025802666942278545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,15,0.025759999950726826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,15,0.021909333765506744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,15,0.02610666553179423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,15,0.025514667232831318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,15,0.02497600018978119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,15,0.02481066683928172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,15,0.02495466669400533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,15,0.024677333732446034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,15,0.02510933329661687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,31,0.026234666506449383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,31,0.02569066733121872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,31,0.02142400046189626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,31,0.02534399926662445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,31,0.024608001112937927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,31,0.02516266703605652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,31,0.024890666206677754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,31,0.025311999022960663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,31,0.020586666961510975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,31,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,31,0.024490666886170704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,31,0.02407466620206833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,31,0.024080000817775726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,31,0.024618667860825855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,31,0.02388266722361247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,63,0.020469332734743755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,31,0.024149333437283833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,63,0.02514133354028066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,63,0.025487999121348064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,63,0.024218666056791942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,63,0.024197332561016083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,63,0.024112001061439514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,63,0.02403733382622401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,63,0.024170666933059692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,63,0.023503998915354412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,63,0.01974933346112569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,63,0.023962666591008503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,63,0.02481066683928172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,63,0.02370133250951767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,63,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,63,0.023765332996845245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,63,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,127,0.02351466566324234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,127,0.027866666515668232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,127,0.02719466636578242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,127,0.026522666215896606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,127,0.026213333010673523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,127,0.026288000245889027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,127,0.026144000391165417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,127,0.026021334032217663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,127,0.027376001079877216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,127,0.022677332162857056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,127,0.02565866708755493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,127,0.026122666895389557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,127,0.025311999022960663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,127,0.025418666501839954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,127,0.025050667424996693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,127,0.025424001117547352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,255,0.023306667804718018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,255,0.02621866762638092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,255,0.02864533414443334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,255,0.02828799933195114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,255,0.026426665484905243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,255,0.02585600068171819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,255,0.026000000536441803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,255,0.022592000663280487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,255,0.026191999514897663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,255,0.02701333413521449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,255,0.025759999950726826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,255,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,255,0.025957333544890087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,255,0.025061334172884624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,255,0.025061334172884624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,255,0.024570666253566742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,511,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,511,0.023818666736284893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,511,0.027232001225153606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,511,0.0286613330245018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,511,0.026416001220544178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,511,0.026357332865397137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,511,0.023936000963052113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,511,0.026170666019121807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,511,0.026474667092164356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,511,0.027727998793125153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,511,0.02606933315594991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,511,0.02586666742960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,511,0.03032533327738444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,511,0.02586666742960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,511,0.02608533451954524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,511,0.02565866708755493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,1023,0.02622933437426885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,1023,0.02756800005833308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,1023,0.029546665648619335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,1023,0.02865600089232127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,1023,0.028117333849271137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,1023,0.02788266787926356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,1023,0.027776000400384266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,1023,0.025994665920734406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,1023,0.02897600084543228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,1023,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,1023,0.027093333502610523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,1023,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,1023,0.0268053337931633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,1023,0.027701333165168762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,1023,0.026928000152111053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,1023,0.02703999976317088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,2047,0.028223998844623566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,2047,0.03310399999221166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,2047,0.031066666046778362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,2047,0.031541332602500916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,2047,0.030239999294281006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,2047,0.029482667644818623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,2047,0.02943466603755951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,2047,0.029701332251230877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,2047,0.028309332827727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,2047,0.03249066571394602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,2047,0.030565333863099415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,2047,0.028959999481836956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,2047,0.028944000601768494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,2047,0.029919999341169994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,2047,0.028864001234372456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,2047,0.028858666618665058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,4095,0.030506665507952373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,4095,0.03550933301448822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,4095,0.03718933214743932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,4095,0.03400533397992452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,4095,0.03275733441114426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,4095,0.03366400053103765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,4095,0.033957332372665405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,4095,0.033615998923778534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,4095,0.030234667162100475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,4095,0.03856533269087473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,4095,0.03455466777086258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,4095,0.03348266581694285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,4095,0.03284800052642822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,4095,0.032885332902272545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,4095,0.03311466674009959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,4095,0.03279466678698858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,8191,0.0352960005402565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,8191,0.03993066648642222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,8191,0.03578133384386698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,8191,0.036533333361148834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,8191,0.036831999818483986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,8191,0.03773866593837738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,8191,0.03696000079313914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,8191,0.03320533285538355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,8191,0.036229332288106285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,8191,0.03824000060558319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,8191,0.03641066700220108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,8191,0.035349334279696144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,8191,0.03439466655254364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,8191,0.0351200004418691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,8191,0.034832000732421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,8191,0.03474666674931844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,16383,0.041322665909926094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,16383,0.05040533343950907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,16383,0.04625066618124644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,16383,0.04446400205294291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,16383,0.04385066529115041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,16383,0.04466133316357931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,16383,0.04437333345413208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,16383,0.03799466788768768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,16383,0.0439573327700297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,16383,0.0473333348830541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,16383,0.04435733457406362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,16383,0.04274133344491323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,16383,0.04279466470082601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,16383,0.04228266576925913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,16383,0.042362665136655174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,16383,0.0420959989229838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,32767,0.05308799942334493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,32767,0.061466669042905174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,32767,0.04975466430187225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,32767,0.05797333518664042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,32767,0.052576000491778054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,32767,0.04880533119042715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,32767,0.04965866605440775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,32767,0.04650133351484934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,32767,0.04915200173854828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,32767,0.05179733534653982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,32767,0.047600001096725464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,32767,0.045978665351867676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,32767,0.04500266909599304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,32767,0.04506133496761322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,32767,0.044826666514078774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,32767,0.04543466866016388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,65535,0.07157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,65535,0.067930668592453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,65535,0.06958933174610138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,65535,0.07606933514277141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,65535,0.07009600102901459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,65535,0.06738666693369548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,65535,0.06724266707897186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,65535,0.059664001067479454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,65535,0.0672159989674886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,65535,0.0626453310251236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,65535,0.05312533179918925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,65535,0.055349335074424744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,65535,0.06073066592216492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,65535,0.05217599868774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,65535,0.05266133447488149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,65535,0.050794666012128196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1,1,4,131071,0.09736532966295879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1,1,2,131071,0.10040533542633057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1,1,1,131071,0.09790933132171631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1,1,8,131071,0.09361066420873006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1,1,16,131071,0.09325866897900899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1,1,32,131071,0.09232532978057861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1,1,64,131071,0.0913866659005483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1,1,128,131071,0.09077866872151692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1,1,1,131071,0.08418132861455281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1,1,2,131071,0.0774239997069041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1,1,4,131071,0.0740533322095871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1,1,8,131071,0.07167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1,1,16,131071,0.07072000205516815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1,1,32,131071,0.07083733379840851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1,1,64,131071,0.0697386662165324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1,1,128,131071,0.0691786656777064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,1,0.02290133386850357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,1,0.026869334280490875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,1,0.027669332921504974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,1,0.027045334378878277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,1,0.02641066660483678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,1,0.02646933247645696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,1,0.026127999027570088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,1,0.026426665484905243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,1,0.02274666726589203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,1,0.026389333109060924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,1,0.026170666019121807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,1,0.027050666511058807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,1,0.02566933383544286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,1,0.025744001070658367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,1,0.025744001070658367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,1,0.025605333348115284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,3,0.02271999915440877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,3,0.027535999814669292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,3,0.026816000541051228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,3,0.026554666459560394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,3,0.02587733417749405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,3,0.02607999990383784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,3,0.02613866577545802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,3,0.022490667800108593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,3,0.026261332134405773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,3,0.02657066782315572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,3,0.025962665677070618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,3,0.026133333643277485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,3,0.02548266698916753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,3,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,3,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,3,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,7,0.022810667753219604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,7,0.026933332284291584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,7,0.027066667874654133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,7,0.02622933437426885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,7,0.025775998830795288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,7,0.025920001169045765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,7,0.026250667870044708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,7,0.02595199892918269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,7,0.021989333132902782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,7,0.026848000784715016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,7,0.026005332668622334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,7,0.025653332471847534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,7,0.025418666501839954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,7,0.025498665869235992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,7,0.02555199960867564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,15,0.022181332111358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,7,0.025173333783944447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,15,0.027024000883102417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,15,0.02638400097688039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,15,0.026575999955336254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,15,0.02587733417749405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,15,0.02589866767326991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,15,0.02593066543340683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,15,0.025818665822347004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,15,0.0220266655087471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,15,0.025983999172846477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,15,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,15,0.024911999702453613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,15,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,15,0.02624000112215678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,15,0.025072000920772552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,15,0.024832000335057575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,31,0.021701333423455555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,31,0.025983999172846477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,31,0.027077332139015198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,31,0.025231999655564625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,31,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,31,0.025034666061401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,31,0.025434667865435284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,31,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,31,0.021402666966120403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,31,0.025781333446502686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,31,0.024501333634058636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,31,0.02508266766866048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,31,0.024149333437283833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,31,0.024666666984558105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,31,0.02420266717672348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,31,0.02402133246262868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,63,0.02554133286078771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,63,0.02186133215824763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,63,0.02470933397610982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,63,0.02496533344189326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,63,0.024143998821576435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,63,0.024656000236670177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,63,0.024245334168275196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,63,0.023984000086784363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,63,0.02027200038234393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,63,0.024933333198229473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,63,0.023631999890009563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,63,0.02342933416366577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,63,0.023823998868465424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,63,0.0236160010099411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,63,0.6916800340016683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,63,0.02314666658639908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,127,0.023775999744733173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,127,0.02697066714366277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,127,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,127,0.028117333849271137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,127,0.026789332429567974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,127,0.026373334228992462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,127,0.02607999990383784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,127,0.026144000391165417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,127,0.023077333966890972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,127,0.027621333797772724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,127,0.026250667870044708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,127,0.025578667720158894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,127,0.0260959987839063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,127,0.025568000972270966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,127,0.025637333591779072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,127,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,255,0.028629332780838013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,255,0.027221334477265675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,255,0.025045332809289295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,255,0.02701866626739502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,255,0.026709333062171936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,255,0.026176000634829204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,255,0.02622933437426885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,255,0.026159999271233875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,255,0.023733332753181458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,255,0.027466667195161183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,255,0.026378666361172993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,255,0.026074667771657307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,255,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,255,0.025834667185942333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,255,0.025407999753952026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,255,0.025616000096003216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,511,0.025727999707063038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,511,0.029264000554879505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,511,0.028218666712443035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,511,0.02754666656255722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,511,0.027535999814669292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,511,0.027280000348885853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,511,0.02679466704527537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,511,0.02716800073782603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,511,0.02884799987077713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,511,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,511,0.02716800073782603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,511,0.026816000541051228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,511,0.027093333502610523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,511,0.02643200010061264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,511,0.02678400029738744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,1023,0.02846933404604594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,511,0.026789332429567974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,1023,0.030586667358875275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,1023,0.03238933285077413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,1023,0.02961066613594691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,1023,0.029509333272775013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,1023,0.02903466671705246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,1023,0.02918400118748347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,1023,0.027914665639400482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,1023,0.03143466760714849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,1023,0.03164266546567281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,1023,0.030031998952229817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,1023,0.02850666642189026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,1023,0.02863466739654541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,1023,0.02861333390076955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,1023,0.02918400118748347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,1023,0.0284853329261144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,2047,0.03625066578388214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,2047,0.0301706666747729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,2047,0.03385599950949351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,2047,0.03297066688537598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,2047,0.03159466634194056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,2047,0.03217600037654241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,2047,0.031701333820819855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,2047,0.03154666721820831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,2047,0.029301332930723827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,2047,0.03264000018437704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,2047,0.03513599932193756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,2047,0.031141333281993866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,2047,0.030965333183606465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,2047,0.03133866687615713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,2047,0.03142933299144109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,2047,0.030949334303538006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,4095,0.03314133236805598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,4095,0.037605332831541695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,4095,0.03644266724586487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,4095,0.034527999659379326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,4095,0.03440000116825104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,4095,0.03432533393303553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,4095,0.03467733412981033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,4095,0.03215466688076655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,4095,0.035973332822322845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,4095,0.03419200082619985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,4095,0.0339626669883728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,4095,0.033013333876927696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,4095,0.03301866600910822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,4095,0.03294933338960012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,4095,0.033088001112143196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,8191,0.04320000112056732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,8191,0.037776000797748566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,8191,0.04221866528193156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,4095,0.03350933392842611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,8191,0.03977066775163015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,8191,0.03958400090535482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,8191,0.03924266745646795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,8191,0.038704000413417816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,8191,0.04087999959786733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,8191,0.038762666285037994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,8191,0.03665599972009659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,8191,0.03875199953715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,8191,0.03687999894221624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,8191,0.0371573343873024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,8191,0.03766400118668874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,8191,0.037077332536379494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,8191,0.0369759996732076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,16383,0.052101333936055504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,16383,0.05859733124574026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,16383,0.04673600196838379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,16383,0.049360002080599465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,16383,0.05426666637261709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,16383,0.04550399879614512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,16383,0.048714667558670044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,16383,0.04568533102671305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,16383,0.047242666284243263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,16383,0.045253331462542214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,16383,0.043375998735427856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,16383,0.04252799848715464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,16383,0.042362665136655174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,16383,0.04236799975236257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,16383,0.04177600145339966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,16383,0.041637333730856575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,32767,0.06886933247248332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,32767,0.06714666883150737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,32767,0.07337066531181335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,32767,0.0668746680021286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,32767,0.0654666672150294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,32767,0.06442666550477345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,32767,0.06438399851322174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,32767,0.0634986658891042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,32767,0.05958400170008341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,32767,0.057946667075157166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,32767,0.056015998125076294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,32767,0.05301333467165629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,32767,0.04976533353328705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,32767,0.04966400067011515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,32767,0.048911998669306435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,32767,0.0492799977461497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,65535,0.09611200292905171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,65535,0.0981173316637675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,65535,0.09788800279299419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,65535,0.09132267038027446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,65535,0.0897813340028127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,65535,0.09195733070373535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,65535,0.0869706670443217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,65535,0.08206399778525035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,65535,0.08923733234405518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,65535,0.07488533357779185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,65535,0.07097599903742473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,65535,0.06799466907978058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,65535,0.06630933284759521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,65535,0.06619733572006226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,65535,0.0695253312587738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,65535,0.06685333450635274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,2,1,2,131071,0.14959999918937683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,2,1,1,131071,0.15343999862670898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,2,1,4,131071,0.14492266376813254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,2,1,8,131071,0.1411733329296112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,2,1,16,131071,0.1402186652024587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,2,1,32,131071,0.13946666320165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,2,1,64,131071,0.13806399703025818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,2,1,128,131071,0.1379200021425883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,2,1,1,131071,0.13155200084050497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,2,1,4,131071,0.09649067123730977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,2,1,2,131071,0.09983467062314351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,2,1,8,131071,0.0946666697661082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,2,1,32,131071,0.09239466985066731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,2,1,16,131071,0.09353599945704143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,2,1,128,131071,0.09237866600354512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,2,1,64,131071,0.09243200222651164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,1,0.022826666633288067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,1,0.027877333263556164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,1,0.027056001126766205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,1,0.02649066597223282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,1,0.02643733223279317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,1,0.026015999416510265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,1,0.02622399975856145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,1,0.0262719988822937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,1,0.027072000006834667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,1,0.02271999915440877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,1,0.02632533262173335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,1,0.02595199892918269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,1,0.025792000194390614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,1,0.025775998830795288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,1,0.025802666942278545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,1,0.026005332668622334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,3,0.027749332288901012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,3,0.02701333413521449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,3,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,3,0.026816000541051228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,3,0.026522666215896606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,3,0.026533332963784535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,3,0.02619733413060506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,3,0.026474667092164356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,3,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,3,0.022874665757020313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,3,0.026762666801611584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,3,0.026133333643277485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,3,0.026191999514897663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,3,0.026261332134405773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,3,0.026000000536441803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,7,0.023071999351183575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,3,0.026000000536441803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,7,0.02790933350721995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,7,0.027306665976842243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,7,0.02664000044266383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,7,0.02657066782315572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,7,0.02638400097688039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,7,0.026362667481104534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,7,0.02643733223279317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,7,0.02717333287000656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,7,0.022863999009132385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,7,0.026144000391165417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,7,0.02569066733121872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,7,0.026005332668622334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,7,0.02552533398071925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,7,0.025514667232831318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,7,0.02553066611289978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,15,0.022522665560245514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,15,0.0260959987839063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,15,0.027136000494162243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,15,0.026501332720120747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,15,0.026186667382717133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,15,0.025770666698614757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,15,0.02624533325433731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,15,0.026213333010673523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,15,0.022106667359670002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,15,0.026746665438016255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,15,0.02571733295917511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,15,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,15,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,15,0.025621332228183746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,15,0.025568000972270966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,15,0.025216000775496166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,31,0.022298666338125866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,31,0.0264533335963885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,31,0.02569066733121872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,31,0.026778665681680042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,31,0.02629333237806956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,31,0.02552533398071925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,31,0.025818665822347004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,31,0.02521066615978877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,31,0.021957332889238994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,31,0.025733334322770435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,31,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,31,0.02477866659561793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,31,0.024559999505678814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,31,0.025018667181332905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,31,0.024800000091393787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,31,0.024351999163627625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,63,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,63,0.02589866767326991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,63,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,63,0.024688000480333965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,63,0.0199946661790212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,63,0.02441066751877467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,63,0.0244159996509552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,63,0.025029333929220837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,63,0.6886400381724039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,63,0.02497066557407379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,63,0.02437866727511088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,63,0.02363733450571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,63,0.024517332514127094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,63,0.02367466688156128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,63,0.02458133300145467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,63,0.023567999402681988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,127,0.02420266717672348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,127,0.02958400050799052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,127,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,127,0.027850667635599773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,127,0.026634665826956432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,127,0.026736001173655193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,127,0.026288000245889027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,127,0.026714667677879333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,127,0.023717333873112995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,127,0.02826133370399475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,127,0.02629866699377696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,127,0.02701866626739502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,127,0.026288000245889027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,127,0.026159999271233875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,127,0.02593066543340683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,127,0.025888000925381977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,255,0.027893332143624622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,255,0.028079998989899952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,255,0.026650667190551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,255,0.029498666524887085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,255,0.027050666511058807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,255,0.026922665536403656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,255,0.027029333015282948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,255,0.0268053337931633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,255,0.028202667832374573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,255,0.02569066733121872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,255,0.026741333305835724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,255,0.02734400083621343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,255,0.026416001220544178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,255,0.02629333237806956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,255,0.02606400102376938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,255,0.0262719988822937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,511,0.031717332700888314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,511,0.02741333345572154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,511,0.03046933313210805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,511,0.028549333413441975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,511,0.028650666276613872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,511,0.02905600021282832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,511,0.02812800059715907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,511,0.028250666956106823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,511,0.029322666426499683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,511,0.027552001178264618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,511,0.03162133445342382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,511,0.0286613330245018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,511,0.027866666515668232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,511,0.0276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,511,0.027808000644048054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,511,0.027674667537212372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,1023,0.03305600086847941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,1023,0.03213333338499069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,1023,0.030576000610987347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,1023,0.03444266567627589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,1023,0.030784000953038532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,1023,0.030799999833106995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,1023,0.031114667654037476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,1023,0.031125334401925404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,1023,0.02956799914439519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,1023,0.03289066751797994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,1023,0.03463999927043915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,1023,0.03129599988460541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,1023,0.03054400036732356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,1023,0.03062933435042699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,1023,0.03035199890534083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,1023,0.030346666773160298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,2047,0.03327466547489166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,2047,0.03536533315976461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,2047,0.03882133215665817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,2047,0.034234667817751564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,2047,0.0337119996547699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,2047,0.03366400053103765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,2047,0.03418133407831192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,2047,0.03321066747109095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,2047,0.03164266546567281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,2047,0.0325546662012736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,2047,0.03682666768630346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,2047,0.033861334125200905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,2047,0.031930667658646904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,2047,0.03183999905983607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,2047,0.032357332607110344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,4095,0.04154666761557261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,2047,0.03232000023126602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,4095,0.03873066604137421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,4095,0.04062400013208389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,4095,0.03814399987459183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,4095,0.03742400060097376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,4095,0.03774400055408478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,4095,0.038890667259693146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,4095,0.03629866739114126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,4095,0.03958933303753535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,4095,0.03775999943415324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,4095,0.03661333272854487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,4095,0.0378506655494372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,4095,0.03568000098069509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,4095,0.035445332527160645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,8191,0.05407999952634176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,8191,0.053727999329566956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,8191,0.047269334395726524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,4095,0.03522666543722153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,4095,0.03580799947182337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,8191,0.052527998884518944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,8191,0.04329599936803182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,8191,0.04357333481311798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,8191,0.04517333209514618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,8191,0.04644800225893656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,8191,0.04348800083001455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,8191,0.04528533418973287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,8191,0.0399893323580424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,8191,0.040864000717798867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,8191,0.04218133290608724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,8191,0.040250666439533234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,8191,0.03978666663169861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,8191,0.04012800008058548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,16383,0.06724266707897186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,16383,0.07132799923419952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,16383,0.06453333298365276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,16383,0.06771733363469441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,16383,0.06480533381303151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,16383,0.061936000982920326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,16383,0.06265600025653839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,16383,0.062090665102005005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,16383,0.05972800155480703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,16383,0.05866133173306783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,16383,0.05458133419354757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,16383,0.047541335225105286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,16383,0.05036266644795736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,16383,0.047269334395726524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,16383,0.04718933502833048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,16383,0.04665599763393402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,32767,0.08986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,32767,0.10312533378601074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,32767,0.09097599983215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,32767,0.09583466251691182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,32767,0.08706667025883992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,32767,0.08773866295814514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,32767,0.08683733145395915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,32767,0.08608532945315044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,32767,0.08089066545168559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,32767,0.07313066720962524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,32767,0.06821333368619283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,32767,0.0664213349421819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,32767,0.0653546651204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,32767,0.06437866886456807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,32767,0.06542933483918507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,32767,0.06481066842873891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,65535,0.1511679987112681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,65535,0.14924266934394836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,65535,0.1399893363316854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,65535,0.14239999651908875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,65535,0.13801067074139914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,65535,0.13700800140698752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,65535,0.1360586682955424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,65535,0.12913599610328674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,65535,0.13619732856750488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,65535,0.0946666697661082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,65535,0.09304533402125041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,65535,0.09924800197283427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,65535,0.09141866366068523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,65535,0.09673066933949788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,65535,0.09036800265312195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,65535,0.09053867061932881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,4,1,1,131071,0.26336532831192017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,4,1,2,131071,0.24649600187937418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,4,1,4,131071,0.240447998046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,4,1,8,131071,0.23698665698369345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,4,1,64,131071,0.2363626758257548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,4,1,16,131071,0.23694399992624918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,4,1,128,131071,0.23556800683339438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,4,1,32,131071,0.2342346707979838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,4,1,1,131071,0.2254133423169454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,4,1,2,131071,0.15494400262832642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,4,1,8,131071,0.14710399508476257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,4,1,4,131071,0.14999999602635702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,4,1,16,131071,0.14500266313552856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,4,1,64,131071,0.14476799964904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,4,1,32,131071,0.1451520025730133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,1,0.023711999257405598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,4,1,128,131071,0.14519466956456503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,1,0.028602667152881622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,1,0.027962667246659596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,1,0.02773333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,1,0.026661333938439686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,1,0.026949333647886913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,1,0.02678400029738744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,1,0.026677332818508148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,1,0.02802666773398717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,1,0.024112001061439514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,1,0.027215999861558277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,1,0.02638400097688039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,1,0.026170666019121807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,1,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,1,0.0260959987839063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,1,0.026213333010673523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,3,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,3,0.028538666665554047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,3,0.027749332288901012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,3,0.026746665438016255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,3,0.02717333287000656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,3,0.026848000784715016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,3,0.026874666412671406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,3,0.026762666801611584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,3,0.024117333193620045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,3,0.026949333647886913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,3,0.028181334336598713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,3,0.026650667190551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,3,0.026575999955336254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,3,0.02629333237806956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,3,0.02644266684850057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,7,0.024170666933059692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,3,0.02643733223279317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,7,0.0272533322374026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,7,0.028368001182874043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,7,0.027130665878454845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,7,0.027877333263556164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,7,0.026506667335828144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,7,0.026741333305835724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,7,0.026709333062171936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,7,0.023941333095232647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,7,0.02681066592534383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,7,0.027493332823117573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,7,0.026634665826956432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,7,0.026693334182103474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,7,0.025983999172846477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,7,0.02643200010061264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,7,0.025968000292778015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,15,0.02334933231274287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,15,0.028351999819278717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,15,0.026762666801611584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,15,0.026719999810059864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,15,0.02731200059254964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,15,0.02622933437426885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,15,0.02346666653951009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,15,0.026208000878492992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,15,0.026629333694775898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,15,0.027850667635599773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,15,0.02595199892918269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,15,0.02601066728432973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,15,0.025754667818546295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,15,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,15,0.025562666356563568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,31,0.02325333406527837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,15,0.026074667771657307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,31,0.027903998891512554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,31,0.027034667630990345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,31,0.026591998835404713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,31,0.02595199892918269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,31,0.025626666843891144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,31,0.025850666066010792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,31,0.026015999416510265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,31,0.022639999787012737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,31,0.025770666698614757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,31,0.025594666600227356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,31,0.026869334280490875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,31,0.024901332954565685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,31,0.025381334125995636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,31,0.025018667181332905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,31,0.025381334125995636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,63,0.02325333406527837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,63,0.026309333741664886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,63,0.02478933334350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,63,0.02605333427588145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,63,0.02535466601451238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,63,0.025061334172884624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,63,0.02459733436505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,63,0.021920000513394673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,63,0.025279998779296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,63,0.025701334079106648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,63,0.025013332565625507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,63,0.024405332903067272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,63,0.02409599969784419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,63,0.02465066562096278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,63,0.023962666591008503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,63,0.023978665471076965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,127,0.026261332134405773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,127,0.029391999046007793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,127,0.02881066749493281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,127,0.02809600035349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,127,0.027664000789324444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,127,0.027658666173617046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,127,0.027552001178264618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,127,0.02719466636578242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,127,0.025792000194390614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,127,0.02848000079393387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,127,0.027632000545660656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,127,0.027493332823117573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,127,0.026890667776266735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,127,0.026682667434215546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,127,0.026661333938439686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,127,0.026762666801611584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,255,0.028442665934562683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,255,0.03189333279927572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,255,0.03047466774781545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,255,0.029520000020662945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,255,0.028991999725500744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,255,0.028351999819278717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,255,0.02888533224662145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,255,0.028399998943010967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,255,0.027802666028340656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,255,0.03162666658560435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,255,0.029461334149042766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,255,0.02889599899450938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,255,0.028389332195123036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,255,0.02738133321205775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,255,0.028079998989899952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,511,0.030576000610987347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,511,0.03291733314593633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,511,0.034602666894594826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,255,0.027664000789324444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,511,0.031744000812371574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,511,0.03108799954255422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,511,0.030879999200503033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,511,0.031034665803114574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,511,0.03088533381621043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,511,0.034559999903043113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,511,0.02995733420054118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,511,0.031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,511,0.031194667021433514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,511,0.03109866629044215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,511,0.029887999097506206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,511,0.030069333811601002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,511,0.029909332593282063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,1023,0.03763733307520548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,1023,0.03377600014209747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,1023,0.03489066660404205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,1023,0.03462400039037069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,1023,0.03416533271471659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,1023,0.033301333586374916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,1023,0.03329599897066752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,1023,0.03313066562016805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,1023,0.03338133295377096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,1023,0.03562666724125544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,1023,0.033370666205883026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,1023,0.03286933402220408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,1023,0.032261334359645844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,1023,0.03152533372243246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,1023,0.03155199935038885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,1023,0.03137599925200144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,2047,0.04305600126584371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,2047,0.03979733337958654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,2047,0.03910933434963226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,2047,0.038693333665529885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,2047,0.04008000095685323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,2047,0.0373279998699824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,2047,0.03786666691303253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,2047,0.2887360056241353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,2047,0.03728000074625015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,2047,0.03712533414363861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,2047,0.036602665980656944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,2047,0.037717332442601524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,2047,0.03583999971548716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,2047,0.035445332527160645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,2047,0.03525333354870478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,4095,0.05342933535575867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,2047,0.035029334326585136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,4095,0.0553653339544932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,4095,0.051039998730023704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,4095,0.04689066608746847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,4095,0.04456533491611481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,4095,0.04295999805132548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,4095,0.04343999922275543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,4095,0.043365334471066795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,4095,0.046112000942230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,4095,0.045168002446492515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,4095,0.04205333193143209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,4095,0.04038399954636892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,4095,0.04140799989302953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,4095,0.03917866696914037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,4095,0.03908800085385641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,4095,0.03929600119590759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,8191,0.07043200234572093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,8191,0.0670826683441798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,8191,0.07116800049940745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,8191,0.06274666885534923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,8191,0.06253866851329803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,8191,0.0637066662311554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,8191,0.06258133550484975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,8191,0.06621866424878438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,8191,0.06187200049559275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,8191,0.05445333321889242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,8191,0.059152002135912575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,8191,0.04987733562787374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,8191,0.04859200119972229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,8191,0.04866133133570353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,8191,0.04821866750717163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,8191,0.048026666045188904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,16383,0.0959999958674113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,16383,0.0953546663125356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,16383,0.09092799822489421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,16383,0.09110933542251587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,16383,0.08802666266759236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,16383,0.08687999844551086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,16383,0.0855466624101003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,16383,0.07353599866231282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,16383,0.08525866270065308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,16383,0.06804800033569336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,16383,0.0830026666323344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,16383,0.06644799808661143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,16383,0.06436799963315327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,16383,0.06437333424886067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,16383,0.06309866905212402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,16383,0.06394133468468984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,32767,0.15543466806411743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,32767,0.1480959951877594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,32767,0.14095999797185263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,32767,0.13993066549301147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,32767,0.13752532998720804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,32767,0.13606933752695718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,32767,0.13664533694585165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,32767,0.13582400480906168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,32767,0.1313866674900055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,32767,0.09285333752632141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,32767,0.09896000226338704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,32767,0.09330667058626811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,32767,0.09109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,32767,0.09107200304667155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,32767,0.0902880032857259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,32767,0.08982400099436443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,65535,0.2454986572265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,65535,0.23825067281723022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,65535,0.2659200032552083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,65535,0.23642667134602866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,65535,0.2350239952405294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,65535,0.2325813372929891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,65535,0.2343626618385315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,65535,0.23307732741038004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,65535,0.2265066703160604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,65535,0.15438933173815408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,65535,0.14918399850527445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,65535,0.14517333110173544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,65535,0.14382933576901755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,65535,0.14628266294797262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,65535,0.14536533753077188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,65535,0.1427893340587616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,8,1,1,131071,0.48332266012827557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,8,1,2,131071,0.4384160041809082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,8,1,4,131071,0.4346933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,8,1,8,131071,0.4304746786753337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,8,1,16,131071,0.4317706823348999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,8,1,128,131071,0.4250719944636027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,8,1,64,131071,0.42551998297373456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,8,1,32,131071,0.42612266540527344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,8,1,2,131071,0.25835732618967694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,8,1,1,131071,0.4137386480967204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,8,1,8,131071,0.24963200092315674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,8,1,4,131071,0.2526986598968506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,8,1,64,131071,0.24714134136835733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,8,1,32,131071,0.5518026749293009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,8,1,16,131071,0.25175466140111286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,8,1,128,131071,0.2465333342552185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,1,0.02697066714366277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,1,0.031040000418821972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,1,0.02871999889612198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,1,0.027690666417280834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,1,0.027466667195161183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,1,0.02826133370399475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,1,0.02828799933195114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,1,0.02805333336194356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,1,0.026752000053723652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,1,0.02916266769170761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,1,0.028677334388097126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,1,0.02743999908367793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,1,0.026975999275843304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,1,0.027210667729377747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,1,0.027615999182065327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,1,0.02807466685771942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,3,0.02657066782315572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,3,0.029637334247430164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,3,0.028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,3,0.02812266598145167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,3,0.029919999341169994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,3,0.02945599953333537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,3,0.02773333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,3,0.028037334481875103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,3,0.026101333399613697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,3,0.030085332691669464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,3,0.02769600103298823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,3,0.027877333263556164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,3,0.027429332335789997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,3,0.027237333357334137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,3,0.027349332968393963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,7,0.02646933247645696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,3,0.027306665976842243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,7,0.029738667110602062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,7,0.02870933214823405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,7,0.027984000742435455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,7,0.027823999524116516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,7,0.027488000690937042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,7,0.027829334139823914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,7,0.027477333943049114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,7,0.029296000798543293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,7,0.02794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,7,0.026650667190551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,7,0.02757866680622101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,7,0.027477333943049114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,7,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,15,0.026021334032217663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,7,0.026554666459560394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,15,0.02902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,15,0.028575999041398365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,7,0.027210667729377747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,15,0.02773333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,15,0.027717334528764088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,15,0.0276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,15,0.027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,15,0.025813333690166473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,15,0.02720533311367035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,15,0.027402666707833607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,15,0.026922665536403656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,15,0.02940800040960312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,15,0.026906666656335194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,15,0.02824000020821889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,31,0.02585600068171819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,15,0.026549334327379864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,15,0.026447998980681103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,31,0.029631999631722767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,31,0.02842666705449422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,31,0.027429332335789997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,31,0.027109332382678986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,31,0.026906666656335194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,31,0.026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,31,0.027002667387326557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,31,0.02492266645034154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,31,0.027813332776228588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,31,0.026528000831604004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,31,0.02643200010061264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,31,0.02622933437426885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,31,0.02609066665172577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,31,0.026202666262785595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,31,0.025792000194390614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,63,0.029706666866938274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,63,0.027690666417280834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,63,0.026693334182103474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,63,0.02622399975856145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,63,0.025759999950726826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,63,0.026000000536441803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,63,0.02585600068171819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,63,0.025424001117547352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,63,0.024506665766239166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,63,0.025424001117547352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,63,0.02720000098148982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,63,0.025920001169045765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,63,0.024885334074497223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,63,0.02473066747188568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,63,0.024773334463437397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,63,0.02489600082238515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,127,0.026821332673231762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,127,0.02976000060637792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,127,0.031034665803114574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,127,0.03266133368015289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,127,0.028981332977612812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,127,0.02884799987077713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,127,0.02943466603755951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,127,0.02606400102376938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,127,0.029093332588672638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,127,0.03262399882078171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,127,0.029114666084448498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,127,0.030005333324273426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,127,0.028416000306606293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,127,0.02828266719977061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,127,0.02845333268245061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,127,0.028021333118279774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,255,0.03454933315515518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,255,0.03267733256022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,255,0.03299200038115183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,255,0.03239466746648153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,255,0.03156266609827677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,255,0.03190399954716364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,255,0.03164800008138021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,255,0.031119999786218006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,255,0.031744000812371574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,255,0.03387200087308884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,255,0.03105599929889043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,255,0.030229332546393078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,255,0.030346666773160298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,255,0.030000001192092896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,255,0.029765332738558452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,511,0.03530666728814443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,511,0.03797333439191183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,511,0.03581333408753077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,255,0.02979733298222224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,511,0.0347680002450943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,511,0.03336533407370249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,511,0.03306666761636734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,511,0.03373866776625315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,511,0.033743999898433685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,511,0.034448000291983284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,511,0.03579200059175491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,511,0.032511999209721885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,511,0.0369759996732076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,511,0.03443199892838796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,511,0.03163733333349228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,511,0.03169599920511246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,511,0.032085334261258446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,1023,0.043712000052134194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,1023,0.0421973317861557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,1023,0.03955733279387156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,1023,0.03938133269548416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,1023,0.03801066676775614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,1023,0.037477334340413414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,1023,0.037045332292715706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,1023,0.03787733366092046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,1023,0.04093866546948751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,1023,0.03942399968703588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,1023,0.03748266647259394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,1023,0.03614933292071024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,1023,0.03626666714747747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,1023,0.035386666655540466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,1023,0.03584533433119456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,1023,0.03575466573238373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,2047,0.05203733344872793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,2047,0.05548266569773356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,2047,0.05673066775004069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,2047,0.047824000318845115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,2047,0.04507199923197428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,2047,0.043663998444875084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,2047,0.04338666796684265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,2047,0.044453332821528115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,2047,0.04164266586303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,2047,0.04625066618124644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,2047,0.04786133269468943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,2047,0.04055999964475632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,2047,0.04004266609748205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,2047,0.03969600051641464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,2047,0.03995733211437861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,2047,0.040074666341145836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,4095,0.07271466652552287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,4095,0.07092800239721934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,4095,0.06746133168538411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,4095,0.06516266862551372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,4095,0.06300800045331319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,4095,0.06505066653092702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,4095,0.0620959997177124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,4095,0.062319998939832054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,4095,0.05971199770768484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,4095,0.06182933350404104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,4095,0.053946668903032936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,4095,0.05496533215045929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,4095,0.04744533201058706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,4095,0.04809600114822388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,4095,0.047877331574757896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,4095,0.045850664377212524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,8191,0.09942400455474854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,8191,0.09452266494433086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,8191,0.09957333405812581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,8191,0.09065066774686177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,8191,0.08989866574605306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,8191,0.08823466300964355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,8191,0.08815466364224751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,8191,0.08910399675369263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,8191,0.08646399776140849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,8191,0.07049599786599477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,8191,0.07696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,8191,0.06804266571998596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,8191,0.06647466619809468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,8191,0.0664213349421819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,8191,0.0652106652657191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,16383,0.15430399775505066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,8191,0.06668800115585327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,16383,0.14244266351064047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,16383,0.14877866705258688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,16383,0.13795199990272522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,16383,0.13749866684277853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,16383,0.14474667112032572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,16383,0.1351573367913564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,16383,0.13460800051689148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,16383,0.1341973344484965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,16383,0.0993226667245229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,16383,0.09506666660308838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,16383,0.09088533123334248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,16383,0.09110400080680847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,16383,0.0909440020720164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,16383,0.09061333537101746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,32767,0.26847465833028156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,16383,0.09032533566157024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,32767,0.24593599637349448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,32767,0.2389226754506429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,32767,0.23644266525904337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,32767,0.23653866847356161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,32767,0.23619733254114786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,32767,0.2327573299407959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,32767,0.2318506638209025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,32767,0.22992533445358276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,32767,0.15587199727694193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,32767,0.14639467000961304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,32767,0.14983466267585754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,32767,0.14429866274197897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,32767,0.14402133226394653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,32767,0.14470932881037393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,32767,0.14300266901652017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,65535,0.4389813343683879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,65535,0.48873066902160645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,65535,0.4312800168991089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,65535,0.4290506839752197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,65535,0.4294559955596924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,65535,0.4267359972000122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,65535,0.4282453457514445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,65535,0.42743468284606934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,65535,0.4147626558939616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,65535,0.26952000459035236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,65535,0.26523200670878094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,65535,0.2611146569252014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,65535,0.2567253311475118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,65535,0.2558666666348775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,65535,0.2565653324127197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,16,1,1,131071,0.9249546527862549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,65535,0.2568959991137187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,16,1,2,131071,0.8270666599273682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,16,1,4,131071,0.8204853534698486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,16,1,8,131071,0.8146186669667562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,16,1,16,131071,0.8152960141499838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,16,1,32,131071,0.8123839696248373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,16,1,64,131071,0.8099093437194824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,16,1,128,131071,0.8109920024871826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,16,1,4,131071,0.458842674891154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,16,1,1,131071,0.7898399829864502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,16,1,2,131071,0.4678560098012288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,16,1,8,131071,0.4577653408050537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,16,1,16,131071,0.4529973268508911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,16,1,32,131071,0.45289599895477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,16,1,64,131071,0.45216532548268634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,16,1,128,131071,0.45256535212198895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,1,0.029663999875386555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,1,0.03383466601371765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,1,0.03051200012365977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,1,0.03126933425664902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,1,0.03312533348798752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,1,0.02977066735426585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,1,0.029887999097506206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,1,0.029930666089057922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,1,0.030053332448005676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,1,0.03299200038115183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,1,0.030559999247392017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,1,0.029904000461101532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,1,0.029663999875386555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,1,0.02956266701221466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,1,0.02937600016593933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,1,0.02934933453798294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,3,0.029802667597929638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,3,0.033439998825391136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,3,0.030645333230495453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,3,0.030997333427270252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,3,0.03009066730737686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,3,0.030165334542592365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,3,0.029626667499542236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,3,0.02994133283694585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,3,0.030165334542592365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,3,0.03264000018437704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,3,0.03036266565322876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,3,0.029338667790095013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,3,0.02908266584078471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,3,0.029861333469549816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,3,0.029696000119050343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,3,0.029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,7,0.029978667696317036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,7,0.03092266619205475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,7,0.03306666761636734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,7,0.029877332349618275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,7,0.029968000948429108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,7,0.03033066789309184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,7,0.029850666721661884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,7,0.03028800090154012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,7,0.02992533395687739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,7,0.03282133241494497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,7,0.03012799968322118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,7,0.029722665747006733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,7,0.031445334355036415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,7,0.029520000020662945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,7,0.029253333806991577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,7,0.0296426663796107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,15,0.02902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,15,0.030933332939942677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,15,0.02958400050799052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,15,0.03270400067170461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,15,0.030069333811601002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,15,0.029504001140594482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,15,0.029482667644818623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,15,0.028912000358104706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,15,0.030229332546393078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,15,0.03046933313210805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,15,0.0322826678554217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,15,0.029605334003766377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,15,0.029290666182835896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,15,0.029317334294319153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,15,0.02914133419593175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,15,0.028991999725500744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,31,0.028666667640209198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,31,0.03203733265399933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,31,0.02914133419593175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,31,0.030106666187445324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,31,0.028783999383449554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,31,0.028778667251269024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,31,0.028789333999156952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,31,0.029120000700155895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,31,0.029445332785447437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,31,0.028581333657105763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,31,0.030794667700926464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,31,0.028570666909217834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,31,0.028223998844623566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,31,0.027850667635599773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,31,0.027962667246659596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,31,0.028325334191322327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,63,0.02848000079393387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,63,0.03195200115442276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,63,0.029296000798543293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,63,0.028218666712443035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,63,0.028368001182874043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,63,0.02792000025510788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,63,0.027727998793125153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,63,0.027647999425729115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,63,0.027973333994547527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,63,0.027610667049884796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,63,0.030799999833106995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,63,0.027893332143624622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,63,0.028618666032950085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,63,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,63,0.02712533374627431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,63,0.02720533311367035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,127,0.03485333422819773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,127,0.029978667696317036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,127,0.03263466556866964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,127,0.03130666663249334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,127,0.031114667654037476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,127,0.03352533280849457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,127,0.03065066784620285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,127,0.030778666337331135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,127,0.02974933385848999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,127,0.03181866556406021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,127,0.02959999938805898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,127,0.02886933336655299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,127,0.028549333413441975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,127,0.028949332733949024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,127,0.030224000414212544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,127,0.02847466617822647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,255,0.03378133227427801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,255,0.04099733382463455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,255,0.03737066686153412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,255,0.03654933224121729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,255,0.035973332822322845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,255,0.036848001182079315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,255,0.03708266715208689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,255,0.035375999907652535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,255,0.032655999064445496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,255,0.039664000272750854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,255,0.03504000107447306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,255,0.03470933437347412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,255,0.03425066669782003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,255,0.03405333310365677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,255,0.03429333368937174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,255,0.03403199960788091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,511,0.050623998045921326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,511,0.04574933151404063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,511,0.03897066662708918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,511,0.039936001102129616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,511,0.039594667653242745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,511,0.03823466598987579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,511,0.03879466652870178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,511,0.04730133215586344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,511,0.03774933268626531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,511,0.04215999941031138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,511,0.03755199909210205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,511,0.03659199923276901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,511,0.035589332381884255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,511,0.03619199991226196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,511,0.035946667194366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,511,0.03623466690381368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,1023,0.05771199862162272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,1023,0.06397333244482677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,1023,0.053029333551724754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,1023,0.045552000403404236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,1023,0.04566933214664459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,1023,0.048197334011395775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,1023,0.04475200176239014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,1023,0.044591998060544334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,1023,0.048570667703946434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,1023,0.05669333537419637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,1023,0.04304533203442892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,1023,0.041264000038305916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,1023,0.04107200105985006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,1023,0.04124266654253006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,1023,0.04393066465854645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,2047,0.07753600180149078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,1023,0.040576001008351646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,2047,0.06939733525117238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,2047,0.07489066819349925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,2047,0.06779733300209045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,2047,0.06746133168538411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,2047,0.06402133405208588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,2047,0.06225066880385081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,2047,0.06275733311971028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,2047,0.06735466420650482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,2047,0.0563679983218511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,2047,0.06329600016276042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,2047,0.05351999898751577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,2047,0.050106664498647056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,2047,0.049072002371152244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,2047,0.04779199759165446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,2047,0.04740266501903534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,4095,0.09315199653307597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,4095,0.10492266217867534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,4095,0.0986400047938029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,4095,0.09064533313115437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,4095,0.08958400289217631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,4095,0.0876533289750417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,4095,0.08711466193199158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,4095,0.08656533559163411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,4095,0.07834133505821228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,4095,0.06858666737874348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,4095,0.09247466921806335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,4095,0.07125866909821828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,4095,0.0672106643517812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,4095,0.06651733318964641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,4095,0.06633066634337108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,4095,0.06629333396752675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,8191,0.15854400396347046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,8191,0.1400213340918223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,8191,0.14985600113868713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,8191,0.14441066980361938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,8191,0.14081066846847534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,8191,0.1383519967397054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,8191,0.13737066586812338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,8191,0.13641599814097086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,8191,0.138373335202535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,8191,0.10296000043551128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,8191,0.09449600179990132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,8191,0.09711466232935588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,8191,0.09293333689371745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,8191,0.09354666868845622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,8191,0.09298666318257649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,8191,0.0920906662940979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,16383,0.2785653273264567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,16383,0.24418665965398154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,16383,0.24214933315912882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,16383,0.23745600382486978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,16383,0.23386667172114053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,16383,0.23465599616368613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,16383,0.23438400030136108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,16383,0.23324799537658691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,16383,0.23546133438746134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,16383,0.15558933218320212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,16383,0.15026666720708212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,16383,0.6779359976450602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,16383,0.14459733168284097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,16383,0.14453867077827454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,16383,0.14356266458829245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,16383,0.14356799920399985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,32767,0.49980799357096356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,32767,0.45370133717854816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,32767,0.4388959805170695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,32767,0.4328106641769409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,32767,0.42983468373616535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,32767,0.4265226523081462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,32767,0.42714667320251465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,32767,0.43006400267283124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,32767,0.26850666602452594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,32767,0.4244319995244344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,32767,0.2583093245824178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,32767,0.2539626757303874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,32767,0.24935466051101685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,32767,0.2516480088233948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,32767,0.25035200516382855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,32767,0.24920000632603964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,65535,0.9325493176778158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,65535,0.8299146493275961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,65535,0.8227307001749674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,65535,0.8155786991119385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,65535,0.8119626839955648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,65535,0.8135146299997965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,65535,0.8092959721883138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,65535,0.8033333619435629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,65535,0.4791040023167928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,65535,0.8184053103129069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,65535,0.4645226796468099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,65535,0.4621280034383138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,65535,0.46810134251912433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,65535,0.45795198281606037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,65535,0.45932265122731525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,65535,0.45713067054748535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,32,1,2,131071,1.604042689005534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,32,1,1,131071,1.8055359522501628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,32,1,4,131071,1.5926987330118816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,32,1,16,131071,1.5840373039245605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,32,1,8,131071,1.585408051808675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,32,1,32,131071,1.585301399230957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,32,1,64,131071,1.5810400644938152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,32,1,128,131071,1.5811999638875325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,32,1,2,131071,0.8852907021840414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,32,1,1,131071,1.5750667254130046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,32,1,4,131071,0.8747999668121338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,32,1,8,131071,0.8709440231323242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,1,0.03660800059636434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,32,1,16,131071,0.8689493338267008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,32,1,32,131071,0.8649973074595133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,1,0.03846933444341024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,1,0.034661332766215004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,1,0.035989334185918175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,32,1,64,131071,0.8648906548817953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,32,1,128,131071,0.8654026985168457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,1,0.03389333436886469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,1,0.033786666889985405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,1,0.03885333240032196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,1,0.03390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,1,0.03346666693687439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,1,0.03912533322970072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,1,0.0352906659245491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,1,0.034458667039871216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,1,0.03366933266321818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,1,0.033573334415753685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,3,0.036357333262761436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,1,0.03458133339881897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,1,0.03341866781314214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,3,0.03828799972931544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,3,0.035760000348091125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,3,0.034714666505654655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,3,0.0341333324710528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,3,0.03397866586844126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,3,0.03342933456103007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,3,0.03860266755024592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,3,0.033861334125200905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,3,0.03540800015131632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,3,0.03839466720819473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,3,0.03419200082619985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,3,0.03387733300526937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,3,0.033520000676314034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,3,0.03297599901755651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,7,0.036415999134381614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,3,0.03281066566705704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,7,0.035616000493367515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,7,0.03830400109291077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,7,0.034272000193595886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,7,0.033957332372665405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,7,0.033557333052158356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,7,0.03383466601371765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,7,0.03917866696914037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,7,0.033488000432650246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,7,0.03871466716130575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,7,0.03568000098069509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,7,0.033813332517941795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,7,0.033759998778502144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,7,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,7,0.033770665526390076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,15,0.03583466758330663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,7,0.033029332756996155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,15,0.03510399907827377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,15,0.038202665746212006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,15,0.0332640012105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,15,0.03391999999682108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,15,0.032960000137488045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,15,0.03291733314593633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,15,0.032672000428040825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,15,0.034143999218940735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,15,0.03882133215665817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,15,0.03368533402681351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,15,0.038506666819254555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,15,0.035530666510264076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,15,0.03320533285538355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,15,0.03301866600910822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,15,0.03277866790692011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,31,0.04266133407751719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,31,0.035904000202814736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,31,0.03516799956560135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,31,0.03306666761636734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,31,0.033359999457995095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,31,0.032933334509531655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,31,0.032586666444937386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,31,0.03253333270549774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,31,0.037690666814645134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,31,0.03436266630887985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,31,0.037647999823093414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,31,0.03275733441114426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,31,0.03258133431275686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,31,0.03204799940188726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,31,0.03190399954716364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,63,0.03472000112136205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,31,0.031744000812371574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,63,0.0344106654326121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,63,0.03757333258787791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,63,0.03275199979543686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,63,0.032255999743938446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,63,0.03187733391920725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,63,0.031370667119820915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,63,0.03154666721820831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,63,0.0369759996732076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,63,0.03313600023587545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,63,0.03690666705369949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,63,0.03233066697915395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,63,0.031045332551002502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,63,0.031632001201311745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,63,0.03046933313210805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,127,0.03937066594759623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,127,0.0373279998699824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,63,0.030597334106763203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,127,0.035242666800816856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,127,0.03689600030581156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,127,0.034389334420363106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,127,0.03409066547950109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,127,0.03417066733042399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,127,0.03508266558249792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,127,0.03878933439652125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,127,0.03489066660404205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,127,0.03817066550254822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,127,0.03349866718053818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,127,0.032613334556420646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,127,0.03243733445803324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,127,0.032773333291212715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,127,0.032074667513370514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,255,0.0410453329483668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,255,0.04417066772778829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,255,0.03998400022586187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,255,0.04046933352947235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,255,0.04158399999141693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,255,0.038319999972979225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,255,0.03915199885765711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,255,0.03849066545565923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,255,0.03885866701602936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,255,0.042581334710121155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,255,0.03789333254098892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,255,0.036320000886917114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,255,0.036330667634805046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,255,0.035877334574858345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,255,0.03543466577927271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,511,0.056602666775385536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,255,0.03552533437808355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,511,0.06614399949709575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,511,0.05845333139101664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,511,0.053120002150535583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,511,0.0487360010544459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,511,0.0484746644894282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,511,0.0484799991051356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,511,0.04841599861780802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,511,0.050842667619387306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,511,0.045093332727750145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,511,0.047925333182017006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,511,0.05977599819501241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,511,0.04423999786376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,511,0.04377066592375437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,511,0.043749332427978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,511,0.04420800010363261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,1023,0.08545600374539693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,1023,0.07183466851711273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,1023,0.07452799876530965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,1023,0.07124799986680348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,1023,0.06961599985758464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,1023,0.07256533205509186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,1023,0.06286400059858958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,1023,0.06677333513895671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,1023,0.07285333176453908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,1023,0.06723733246326447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,1023,0.06287999947865804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,1023,0.05759466687838236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,1023,0.056757330894470215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,1023,0.054431999723116554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,1023,0.05322133501370748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,1023,0.05268266797065735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,2047,0.10062399506568909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,2047,0.10764267047246297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,2047,0.09973333279291789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,2047,0.09246399998664856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,2047,0.09479999542236328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,2047,0.091839998960495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,2047,0.09139733513196309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,2047,0.09123200178146362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,2047,0.07693333427111308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,2047,0.08807999889055888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,2047,0.08507200082143147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,2047,0.07250133156776428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,2047,0.07072000205516815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,2047,0.06994133194287618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,2047,0.06894933183987935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,2047,0.06929066777229309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,4095,0.15743999679883322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,4095,0.15678933262825012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,4095,0.1487573285897573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,4095,0.14380266269048056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,4095,0.1406773328781128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,4095,0.1420693298180898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,4095,0.14203199744224548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,4095,0.1428000032901764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,4095,0.13613866766293845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,4095,0.11171199878056844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,4095,0.1036906639734904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,4095,0.09745066364606221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,4095,0.0965280036131541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,4095,0.0974026620388031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,4095,0.09883200128873189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,8191,0.2685546676317851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,4095,0.09748799602190654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,8191,0.2464639941851298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,8191,0.25457600752512616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,8191,0.23900800943374634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,8191,0.2404693365097046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,8191,0.23899734020233154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,8191,0.23690134286880493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,8191,0.23883734146753946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,8191,0.23413866758346558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,8191,0.16460800170898438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,8191,0.15416533748308817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,8191,0.15200000007947287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,8191,0.14732266465822855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,8191,0.147189329067866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,8191,0.14777066310246786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,8191,0.14521599809328714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,16383,0.4451520045598348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,16383,0.5173173348108927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,16383,0.4590453306833903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,16383,0.439797321955363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,16383,0.4333920081456502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,16383,0.4331039985020955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,16383,0.4347519874572754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,16383,0.43429334958394367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,16383,0.2599946657816569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,16383,0.43773333231608075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,16383,0.2818933327992757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,16383,0.25728533665339154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,16383,0.25521600246429443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,16383,0.26871999104817706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,16383,0.25440533955891925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,16383,0.25124800205230713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,32767,0.9550453027089437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,32767,0.8524426619211832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,32767,0.8298346996307373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,32767,0.8215200106302897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,32767,0.8217706680297852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,32767,0.8143839836120605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,32767,0.8153759638468424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,32767,0.8216959635416666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,32767,0.8221706549326578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,32767,0.4876319964726766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,32767,0.47281599044799805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,32767,0.4616159995396932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,32767,0.4597546656926473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,32767,0.4570346673329671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,32767,0.45874667167663574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,32767,0.4578346808751424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,65535,1.825925350189209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,65535,1.6176106135050456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,65535,1.592026710510254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,65535,1.59769074122111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,65535,1.5847573280334473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,65535,1.582032044728597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,65535,1.5854986508687336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,65535,1.5820053418477376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,65535,1.5430933634440105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,65535,0.9000746409098307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,65535,0.8869493007659912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,65535,0.8752640088399252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,65535,0.8700640201568604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,65535,0.8671466509501139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,65535,0.8727359771728516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,65535,0.8649813334147135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,64,1,1,131071,3.562608083089193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,64,1,4,131071,3.135493278503418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,64,1,2,131071,3.15120538075765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,64,1,16,131071,3.124352137247721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,64,1,8,131071,3.1298348108927407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,64,1,64,131071,3.117472012837728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,64,1,128,131071,3.1171627044677734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,64,1,32,131071,3.1241493225097656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,64,1,1,131071,3.019205411275228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,64,1,4,131071,1.7016107241312664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,64,1,2,131071,1.7151360511779785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,64,1,8,131071,1.69048007329305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,64,1,16,131071,1.6916106541951497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,1,0.054144000013669334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,64,1,32,131071,1.6883786519368489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,64,1,64,131071,1.6832906405131023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,64,1,128,131071,1.6918293635050456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,1,0.6832693417867025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,1,0.04362666606903076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,1,0.04182399809360504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,1,0.04062933226426443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,1,0.04008000095685323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,1,0.040175999204317726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,1,0.040048000713189445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,1,0.049957334995269775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,1,0.06411199768384297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,1,0.04424533247947693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,1,0.04136000076929728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,1,0.04074666649103165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,1,0.03966933240493139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,1,0.039877332746982574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,1,0.03985599925120672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,3,0.054655998945236206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,3,0.047637333472569786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,3,0.04177600145339966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,3,0.04010133445262909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,3,0.04379733403523763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,3,0.04035199930270513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,3,0.040847999354203544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,3,0.04049599915742874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,3,0.06355200211207072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,3,0.0498986691236496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,3,0.04125333329041799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,3,0.044106667240460716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,3,0.039994666973749794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,3,0.03949866692225138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,3,0.04018666595220566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,3,0.0399893323580424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,7,0.05346133311589559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,7,0.04729066789150238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,7,0.04359466830889384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,7,0.04005333284536997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,7,0.040362666050593056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,7,0.04171733558177948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,7,0.04057066639264425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,7,0.0632479985555013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,7,0.05018133421738943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,7,0.042549331982930504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,7,0.04456533491611481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,7,0.04146666576464971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,7,0.04268800218900045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,7,0.039503999054431915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,7,0.040394666294256844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,7,0.03986666599909464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,15,0.05553600192070007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,15,0.04712533454100291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,15,0.040991999208927155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,15,0.043578664461771645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,15,0.039546666045983635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,15,0.039477333426475525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,15,0.040778666734695435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,15,0.039264000952243805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,15,0.04990399877230326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,15,0.06383466720581055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,15,0.0444160004456838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,15,0.041237334410349526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,15,0.03980266551176707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,15,0.04054400076468786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,15,0.03965333352486292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,15,0.03941333293914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,31,0.057189335425694786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,31,0.04295999805132548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,31,0.04674133161703745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,31,0.04065600037574768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,31,0.039642666776975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,31,0.03925866633653641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,31,0.03905066599448522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,31,0.06198399762312571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,31,0.03898133337497711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,31,0.03979199876387914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,31,0.04886400202910105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,31,0.042912001411120095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,31,0.03887466589609782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,31,0.03819733361403147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,31,0.03851733356714249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,31,0.038373333712418876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,63,0.06039999922116598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,63,0.046997333566347756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,63,0.042133331298828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,63,0.03990400085846583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,63,0.039093332986036934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,63,0.03798400113979975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,63,0.03856533269087473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,63,0.06388266881306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,63,0.0384853333234787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,63,0.041946664452552795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,63,0.04865066707134247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,63,0.03939733405907949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,63,0.038362666964530945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,63,0.037263999382654824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,63,0.037471999724706016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,63,0.03754666695992152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,127,0.052933335304260254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,127,0.0661653329928716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,127,0.045647998650868736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,127,0.04316799839337667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,127,0.042634665966033936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,127,0.04116799930731455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,127,0.041178666055202484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,127,0.04164800047874451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,127,0.06638933221499126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,127,0.05036266644795736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,127,0.03977066775163015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,127,0.043925335009892784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,127,0.04101333270470301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,127,0.03925333420435587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,127,0.039333333571751915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,127,0.039461334546407066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,255,0.0743146687746048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,255,0.0651146670182546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,255,0.04725866516431173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,255,0.05914133290449778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,255,0.04620266457398733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,255,0.05269333223501841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,255,0.0461706668138504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,255,0.5639466842015585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,255,0.0461706668138504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,255,0.05778133372465769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,255,0.0469813346862793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,255,0.04307200014591217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,255,0.04375466704368591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,255,0.042026668787002563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,255,0.04215466479460398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,255,0.04203199843565623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,511,0.0900320013364156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,511,0.07713599999745686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,511,0.07148266832033794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,511,0.06640000144640605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,511,0.06885333359241486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,511,0.06725333134333293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,511,0.06554133196671803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,511,0.06691200037797292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,511,0.08396800359090169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,511,0.060005332032839455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,511,0.049829334020614624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,511,0.06673066814740498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,511,0.05474133292833964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,511,0.049509331583976746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,511,0.04867733518282572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,511,0.04921066761016846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,1023,0.11899733543395996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,1023,0.09316800038019817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,1023,0.10286399722099304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,1023,0.09605333209037781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,1023,0.0902880032857259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,1023,0.10770133137702942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,1023,0.09026133020718892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,1023,0.09040000041325887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,1023,0.09033600489298503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,1023,0.08082133531570435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,1023,0.07361066838105519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,1023,0.06823466718196869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,1023,0.07029866675535838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,1023,0.06720533470312755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,1023,0.06870399912198384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,1023,0.06674666702747345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,2047,0.1779093345006307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,2047,0.14352533221244812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,2047,0.15330666303634644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,2047,0.14718400438626608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,2047,0.13990933696428934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,2047,0.14124266306559244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,2047,0.14029332995414734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,2047,0.13990933696428934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,2047,0.15542399883270264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,2047,0.1067680021127065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,2047,0.10035199920336406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,2047,0.09570133686065674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,2047,0.09300266702969869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,2047,0.09464533130327861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,2047,0.09497066338857015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,2047,0.09276266892751057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,4095,0.28793599208196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,4095,0.24985599517822266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,4095,0.23990400632222494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,4095,0.23804267247517905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,4095,0.2430079976717631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,4095,0.23732266823450723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,4095,0.23739200830459595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,4095,0.23761600255966187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,4095,0.25332800547281903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,4095,0.15916799505551657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,4095,0.15523733695348105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,4095,0.1495253344376882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,4095,0.14784000317255655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,4095,0.14717866977055868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,4095,0.14528533816337585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,4095,0.14620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,8191,0.5342026551564535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,8191,0.4431360165278117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,8191,0.4346880118052165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,8191,0.43610668182373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,8191,0.442629337310791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,8191,0.43217066923777264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,8191,0.4320586522420247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,8191,0.4352213144302368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,8191,0.45254401365915936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,8191,0.26315732796986896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,8191,0.2577066620190938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,8191,0.2540000081062317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,8191,0.24997333685557047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,8191,0.2504693269729614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,8191,0.24980799357096353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,8191,0.2497333288192749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,16383,0.9723146756490072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,16383,0.8706346352895101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,16383,0.8465973536173502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,16383,0.8310933113098145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,16383,0.8243573506673177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,16383,0.820197343826294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,16383,0.8168426354726156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,16383,0.8238346576690674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,16383,0.8186079661051432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,16383,0.4870026508967082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,16383,0.5101866722106934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,16383,0.4750933249791463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,16383,0.4676373402277629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,16383,0.4599786599477132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,16383,0.46331199010213214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,16383,0.4611946741739909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,32767,1.8406400680541992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,32767,1.6138505935668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,32767,1.6429759661356609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,32767,1.5909387270609539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,32767,1.6172213554382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,32767,1.5857226053873699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,32767,1.5839893023173015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,32767,1.583247979482015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,32767,1.5757120450337727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,32767,0.9255092938741049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,32767,0.9079733689626058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,32767,0.8856906890869141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,32767,0.8759786287943522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,32767,0.874725341796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,32767,0.872986634572347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,32767,0.8691360155741373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,65535,3.1775999069213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,65535,3.5809758504231772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,65535,3.150837262471517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,65535,3.132373491923014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,65535,3.159674644470215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,65535,3.118645350138346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,65535,3.1151040395100913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,65535,3.12006409962972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,65535,1.7481759389241536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,65535,3.0525280634562173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,65535,1.7196586926778157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,65535,1.7032000223795574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,65535,1.6927893956502278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,65535,1.6904640197753906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,65535,1.6862932840983074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,65535,1.6875893274943035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,128,1,4,131071,6.215301513671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,128,1,1,131071,7.040618896484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,128,1,2,131071,6.238431930541992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,128,1,8,131071,6.1956532796223955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,128,1,16,131071,6.258752187093099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,128,1,32,131071,6.1830183664957685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,128,1,64,131071,6.18235715230306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,128,1,128,131071,6.184602737426758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,128,1,2,131071,3.370485305786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,128,1,8,131071,3.333317438761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,128,1,4,131071,3.343455950419108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,128,1,1,131071,5.9886932373046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,128,1,16,131071,3.3263572057088218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,128,1,32,131071,3.321791966756185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,128,1,64,131071,3.3140640258789062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,1,0.09385066231091817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,128,1,128,131071,3.3117758433024087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,1,0.06858666737874348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,1,0.058975999553998314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,1,0.0555626650651296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,1,0.05283733208974203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,1,0.05425600210825602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,1,0.05322133501370748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,1,0.0788213312625885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,1,0.053210665782292686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,1,0.11052266756693523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,1,0.06060799956321716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,1,0.05579199890295664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,1,0.05237866441408793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,1,0.05262400209903717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,1,0.052671998739242554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,1,0.053642665346463524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,3,0.09393067161242168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,3,0.06948799888292949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,3,0.0588266650835673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,3,0.05417599777380625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,3,0.05522666871547699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,3,0.053413331508636475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,3,0.05268799761931101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,3,0.1109226644039154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,3,0.05253866811593374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,3,0.07795199751853943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,3,0.055786664287249245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,3,0.06055466830730438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,3,0.053247998158137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,3,0.05261866748332977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,3,0.0518453319867452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,7,0.06986666719118755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,3,0.05202133456865946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,7,0.09325866897900899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,7,0.05880533158779144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,7,0.055386667450269066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,7,0.053685332338015236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,7,0.05230399966239929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,7,0.05305600166320801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,7,0.052426666021347046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,7,0.10946133732795715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,7,0.07840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,7,0.061717331409454346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,7,0.05314666529496511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,7,0.05429333448410034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,7,0.052655999859174095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,7,0.056549335519472756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,7,0.052341332038243614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,15,0.09302933017412822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,15,0.07142400244871776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,15,0.054570664962132774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,15,0.058149332801500954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,15,0.052986666560173035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,15,0.05131733417510986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,15,0.05180799961090088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,15,0.05190933247407278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,15,0.10974400242169698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,15,0.061424002051353455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,15,0.07778133451938629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,15,0.055359999338785805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,15,0.05348266661167145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,15,0.05226133267084757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,31,0.07471466561158498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,15,0.051872000098228455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,15,0.051957334081331887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,31,0.0958079993724823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,31,0.053226664662361145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,31,0.057146668434143066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,31,0.05223466455936432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,31,0.05089599887530009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,31,0.05081599950790405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,31,0.05852800110975901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,31,0.10812266667683919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,31,0.050981332858403526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,31,0.07614399989446004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,31,0.05279466509819031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,31,0.0492799977461497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,31,0.05091199775536855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,31,0.04991999765237173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,31,0.04937600096066793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,63,0.06090133388837179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,63,0.1002453366915385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,63,0.07813866436481476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,63,0.05268266797065735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,63,0.05113600194454193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,63,0.05023466547330221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,63,0.04966933528582255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,63,0.1079360047976176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,63,0.078575998544693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,63,0.04942933221658071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,63,0.04897599915663401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,63,0.05108266572157542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,63,0.058176000912984215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,63,1.0098506609598796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,63,0.04709866642951965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,63,0.04714133342107137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,127,0.07342933118343353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,127,0.08412800232569377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,127,0.10773866375287373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,127,0.06673599779605865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,127,0.06368533273537953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,127,0.05611200133959452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,127,0.05598933498064677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,127,0.11708266536394756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,127,0.05596800148487091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,127,0.08181866506735484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,127,0.051829333106676735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,127,0.05438933273156484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,127,0.06565866867701213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,127,0.05091199775536855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,127,0.050341332952181496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,127,0.05101866523424784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,255,0.09805867075920105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,255,0.08658132950464885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,255,0.07813866436481476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,255,0.12029866377512614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,255,0.07961600025494893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,255,0.07599466542402904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,255,0.07585066556930542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,255,0.07523199915885925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,255,0.12560000022252402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,255,0.0897653301556905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,255,0.07491733133792877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,255,0.06710933148860931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,255,0.05749333401521047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,255,0.05700799822807312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,511,0.15039466818173727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,255,0.06293866535027821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,255,0.05901333192984263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,511,0.12545067071914673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,511,0.10591999689737956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,511,0.11154133081436157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,511,0.09975999593734741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,511,0.09938133756319682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,511,0.10267200072606404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,511,0.09943466385205586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,511,0.1490720013777415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,511,0.08142933249473572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,511,0.1053546667098999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,511,0.08902933200200398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,511,0.07643199960390727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,511,0.07764799892902374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,511,0.07567999760309856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,1023,0.20908800760904947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,511,0.07619200150171916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,1023,0.16013866662979126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,1023,0.174234668413798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,1023,0.15324800213178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,1023,0.150736004114151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,1023,0.14848533272743225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,1023,0.1472640037536621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,1023,0.13277332981427512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,1023,0.14822399616241455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,1023,0.194650669892629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,1023,0.116047998269399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,1023,0.10948266585667928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,1023,0.10587199529012044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,1023,0.10218133529027303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,1023,0.10340799887975057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,1023,0.10309333602587382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,2047,0.2702346642812093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,2047,0.32222400108973187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,2047,0.256602664788564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,2047,0.2532266577084859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,2047,0.24567466974258423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,2047,0.2486400008201599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,2047,0.2435093323389689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,2047,0.24381333589553833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,2047,0.18535999457041422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,2047,0.15873600045839945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,2047,0.29073067506154376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,2047,0.16942399740219116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,2047,0.16223466396331787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,2047,0.1562026639779409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,2047,0.1562986671924591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,2047,0.15618133544921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,4095,0.4663999875386556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,4095,0.5625866651535034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,4095,0.4436906576156616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,4095,0.45212264855702716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,4095,0.4412959814071655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,4095,0.4402559995651245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,4095,0.4830186764399211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,4095,0.43891199429829914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,4095,0.43940265973409015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,4095,0.914469321568807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,4095,0.2865813374519348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,4095,0.2726400097211202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,4095,0.26052266359329224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,4095,0.2574613293011983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,4095,0.2579893271128337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,4095,0.2578773299853007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,8191,1.0084213415781658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,8191,0.8301546573638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,8191,0.901413361231486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,8191,0.8329226970672607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,8191,0.8470613161722819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,8191,0.8622079690297445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,8191,0.825920025507609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,8191,0.8237760066986084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,8191,0.8689866860707601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,8191,0.5337333281834921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,8191,0.5035039981206259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,8191,0.4877493381500244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,8191,0.47270933787027997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,8191,0.4747680028279622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,8191,0.47091201941172284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,8191,0.46989866097768146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,16383,1.6688159306844075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,16383,1.8871359825134277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,16383,1.6153066953023274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,16383,1.6573066711425781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,16383,1.6002666155497234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,16383,1.5933386484781902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,16383,1.5946666399637859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,16383,1.5912586847941081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,16383,1.6101439793904622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,16383,0.9545706907908121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,16383,0.9206133683522543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,16383,0.9034506479899088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,16383,0.8907626469930013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,16383,0.8815786838531494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,16383,0.8783573309580485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,16383,0.8779146671295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,32767,3.618917465209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,32767,3.2066612243652344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,32767,3.1849705378214517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,32767,3.1742932001749673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,32767,3.135157267252604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,32767,3.1290505727132163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,32767,3.1278454462687173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,32767,3.1272106170654297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,32767,3.126357396443685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,32767,1.782101313273112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,32767,1.742095947265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,32767,1.7217973073323567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,32767,1.7080000241597493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,32767,1.6978613535563152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,32767,1.711045265197754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,32767,1.6945865948994954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,256,1,1,65535,7.067429224650065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,256,1,4,65535,6.233290354410808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,256,1,2,65535,6.3430131276448565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,256,1,8,65535,6.209360122680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,256,1,32,65535,6.192853291829427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,256,1,16,65535,6.2006988525390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,256,1,64,65535,6.188597361246745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,256,1,128,65535,6.1956532796223955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,256,1,2,65535,3.4159412384033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,256,1,1,65535,5.9927628835042315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,256,1,16,65535,3.3286825815836587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,256,1,32,65535,3.3247626622517905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,256,1,64,65535,3.321296056111654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,1,0.15958399573961893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,256,1,128,65535,3.327301343282064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,256,1,4,65535,3.3771839141845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,1,0.11401066184043884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,256,1,8,65535,3.354192097981771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,1,0.09363733728726704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,1,0.08275733391443889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,1,0.0784800002972285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,1,0.07812266548474629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,1,0.07956799864768982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,1,0.07728533446788788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,1,0.19714667399724325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,1,0.10230400164922078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,1,0.13191466530164084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,1,0.08413867155710857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,1,0.07687999804814656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,1,0.0782293329636256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,3,0.15944533546765646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,1,0.07635200023651123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,1,0.07976533472537994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,3,0.09340799848238628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,3,0.11358933647473653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,3,0.082997332016627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,3,0.07980800171693166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,3,0.07893866797288258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,3,0.07797866563002269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,3,0.07749333480993907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,3,0.19608533382415771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,3,0.130431999762853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,3,0.08371200164159139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,3,0.1011786659558614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,3,0.07934399942557017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,3,0.07717866698900859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,3,0.07595733304818471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,3,0.0780213326215744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,7,0.11372799674669902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,7,0.09272533655166626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,7,0.08196266492207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,7,0.07860800127188365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,7,0.15974400440851846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,7,0.0784800002972285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,7,0.07733333110809326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,7,0.07752533257007599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,7,0.12965866923332214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,7,0.10077333450317383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,7,0.19344000021616617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,7,0.0844586690266927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,7,0.07868266602357228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,7,0.0804319977760315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,7,0.07708266874154408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,7,0.07711466650168101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,15,0.157258669535319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,15,0.1149120032787323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,15,0.0802346666653951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,15,0.09244267145792644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,15,0.077674667040507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,15,0.0763733337322871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,15,0.07563200096289317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,15,0.07835199932257335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,15,0.12868266304334006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,15,0.19137599070866904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,15,0.09947733084360759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,15,0.0842186709245046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,15,0.07901333272457123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,15,0.07646933197975159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,15,0.07528533538182576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,31,0.16286933422088623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,15,0.07689600189526875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,31,0.11764267086982727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,31,0.09516800443331401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,31,0.0812853326400121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,31,0.07601599891980489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,31,0.07356266677379608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,31,0.07482666770617168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,31,0.07286933561166127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,31,0.12713600198427835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,31,0.09591466188430786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,31,0.18673600753148398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,31,0.0798880010843277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,31,0.0721919983625412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,31,0.07351466516653697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,31,0.07025066514809926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,31,0.07077866792678833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,63,0.16538133223851523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,63,0.10019733508427937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,63,0.08955199519793193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,63,0.1218346655368805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,63,0.08423999945322673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,63,0.07896000146865845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,63,0.07267733414967854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,63,0.0726506660381953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,63,0.12940800189971924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,63,0.18965866168340048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,63,0.09750399986902873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,63,0.07008533179759979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,63,0.0673226664463679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,63,0.08032533526420593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,63,0.06614933411280315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,63,0.06594133377075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,127,0.18691732486089072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,127,0.13514666755994162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,127,0.10156800349553426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,127,0.11242666840553284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,127,0.09702933828035991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,127,0.09248532851537068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,127,0.0958026647567749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,127,0.09404800335566203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,127,0.2048106590906779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,127,0.10403733452161153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,127,0.13570666313171387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,127,0.08182399968306224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,127,0.08938133716583252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,127,0.07878399888674419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,127,0.0749120016892751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,255,0.2146079937616984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,127,0.07420266668001811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,255,0.16013333201408386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,255,0.1365333298842112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,255,0.12550399700800577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,255,0.11761066317558289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,255,0.12033067146937053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,255,0.11501866579055786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,255,0.11477333307266235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,255,0.22718934218088785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,255,0.1184266706307729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,255,0.09690133730570476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,255,0.10328533252080281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,255,0.0920799970626831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,255,0.0941439966360728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,255,0.09211200475692749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,255,0.14990933736165366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,511,0.2117919921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,511,0.2723520000775655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,511,0.18678933382034302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,511,0.16811732451121011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,511,0.1750026742617289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,511,0.16477333505948386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,511,0.1621280014514923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,511,0.16637333234151205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,511,0.26894400517145794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,511,0.13179199894269308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,511,0.14614933729171753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,511,0.177946666876475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,511,0.12531733512878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,511,0.12258666753768921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,511,0.11932800213495891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,511,0.12176000078519185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,1023,0.39049601554870605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,1023,0.30993600686391193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,1023,0.28406933943430585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,1023,0.2646560072898865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,1023,0.2712799906730652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,1023,0.26208533843358356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,1023,0.25756265719731647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,1023,0.2592800060908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,1023,0.36606399218241376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,1023,0.2333013415336609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,1023,0.20246400435765585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,1023,0.17498666048049927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,1023,0.1787359913190206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,1023,0.1727679967880249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,1023,0.17521067460378012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,2047,0.6210613250732422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,1023,0.18746666113535562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,2047,0.47815465927124023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,2047,0.5042986472447714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,2047,0.46585599581400555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,2047,0.46084264914194745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,2047,0.4525386492411296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,2047,0.45498132705688477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,2047,0.5522453387578329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,2047,0.33488531907399494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,2047,0.45295464992523193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,2047,0.30797332525253296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,2047,0.2911413311958313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,2047,0.2779039939244588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,2047,0.2833813428878784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,2047,0.28065599997838336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,2047,0.2762719988822937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,4095,0.9261653423309326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,4095,1.074453353881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,4095,0.8974133332570394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,4095,0.8703093528747559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,4095,0.8550933202107748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,4095,0.8465920289357504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,4095,0.8403306802113851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,4095,0.8378612995147705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,4095,0.9215786457061768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,4095,0.5766239960988363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,4095,0.5141439835230509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,4095,0.5392586787541708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,4095,0.4914240042368571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,4095,0.49297598997751874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,4095,0.5012906789779663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,4095,0.4868053197860718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,8191,1.9564852714538574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,8191,1.6677385965983074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,8191,1.7325065930684407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,8191,1.6370986302693684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,8191,1.6241013209025066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,8191,1.6123840014139812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,8191,1.60808531443278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,8191,1.6018826166788738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,8191,1.6677813529968262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,8191,1.0036373138427734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,8191,0.9547573725382487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,8191,0.9108426570892334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,8191,0.9250079790751139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,8191,0.8970879713694254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,8191,0.8973120053609213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,8191,0.8974400361378988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,16383,3.705530802408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,16383,3.2511412302652993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,16383,3.171130816141764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,16383,3.241365432739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,16383,3.1497119267781577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,16383,3.1578451792399087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,16383,3.1451520919799805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,16383,3.1462294260660806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,16383,3.1534719467163086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,16383,1.8365492820739746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,16383,1.77894926071167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,16383,1.7447999318440754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,16383,1.7253920237223308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,16383,1.7220800717671711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,16383,1.7122027079264324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,16383,1.7145919799804688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,512,1,4,32767,6.273349126180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,512,1,1,32767,7.200784047444661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,512,1,2,32767,6.326346715291341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,512,1,8,32767,6.318314870198567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,512,1,16,32767,6.226992289225261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,512,1,128,32767,6.207893371582031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,512,1,32,32767,6.2083892822265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,512,1,64,32767,6.208581288655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,512,1,2,32767,3.474229176839193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,512,1,4,32767,3.4098774592081704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,512,1,8,32767,3.387125333150228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,512,1,1,32767,6.075183868408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,512,1,16,32767,3.3600266774495444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,512,1,64,32767,3.342048009236654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,512,1,32,32767,3.3540318806966147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,512,1,128,32767,3.3401333491007485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,1,0.29811733961105347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,1,0.19329599539438883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,1,0.14245333274205527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,1,0.15683733423550925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,1,0.1325386663277944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,1,0.13011200229326883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,1,0.22763200600941977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,1,0.154858668645223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,1,0.37486934661865234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,1,0.12957866986592612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,1,0.1288320024808248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,1,0.13335999846458435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,1,0.17706133921941122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,1,0.12899733583132425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,1,0.12761599818865457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,1,0.1276479959487915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,3,0.2916853427886963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,3,0.19158933560053507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,3,0.14096533258756003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,3,0.15801067153612772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,3,0.1276800036430359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,3,0.13179199894269308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,3,0.12843199570973715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,3,0.12744533022244772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,3,0.36902399857838947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,3,0.2295466661453247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,3,0.13408000270525613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,3,0.14949333667755127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,3,0.17563199996948242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,3,0.12827733159065247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,3,0.126309335231781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,3,0.1253653367360433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,7,0.19176000356674194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,7,0.28987733523050946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,7,0.15715733170509338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,7,0.1397226651509603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,7,0.13026133179664612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,7,0.22741333643595377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,7,0.1258133351802826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,7,0.12688000003496805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,7,0.1264479955037435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,7,0.36230401198069256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,7,0.17573867241541544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,7,0.1342080036799113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,7,0.12943999965985617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,7,0.14820266763369241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,7,0.12627200285593668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,15,0.28909866015116376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,7,0.1269706686337789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,15,0.19106133778889975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,15,0.1536960005760193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,15,0.1384213368097941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,15,0.1304800013701121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,15,0.12479466199874878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,15,0.12309333682060242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,15,0.12310933073361714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,15,0.359877347946167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,15,0.1476853291193644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,15,0.22311999400456747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,15,0.1725333333015442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,15,0.13101866841316223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,15,0.12408533692359924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,15,0.1309760014216105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,31,0.2916319966316223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,15,0.12356799840927124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,31,0.15480533242225647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,31,0.19562667608261108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,31,0.13638933499654135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,31,0.12757333119710287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,31,0.1241973340511322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,31,0.12271466851234436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,31,0.12202133735020955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,31,0.2204266587893168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,31,0.13569600383440653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,31,0.1648960014184316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,31,0.3489493529001872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,31,0.11538666486740112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,31,0.1228053371111552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,31,0.11365333199501038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,31,0.11397332946459453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,63,0.30238399902979535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,63,0.20559465885162354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,63,0.14076266686121622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,63,0.16120533148447672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,63,0.1251200040181478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,63,0.12778133153915405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,63,0.13134400049845377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,63,0.12411200006802876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,63,0.35706134637196857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,63,0.2237386703491211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,63,0.16505600015322366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,63,0.11329066753387451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,63,0.13424533605575562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,63,0.12827199697494507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,63,0.1113973359266917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,63,0.10773866375287373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,127,0.34578665097554523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,127,0.5094186862309774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,127,0.16314133008321127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,127,0.18692266941070557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,127,0.1487573285897573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,127,0.15362133582433066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,127,0.14552533626556396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,127,0.14330666263898215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,127,0.1776640017827352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,127,0.14772799611091614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,127,0.2396906614303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,127,0.3919680118560791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,127,0.1344533363978068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,127,0.12503467003504434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,127,0.12787733475367227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,255,0.3963786760965983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,255,0.23399466276168823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,255,0.20260266462961832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,255,0.28481600681940716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,255,0.21158933639526367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,127,0.12356799840927124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,255,0.19221333662668863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,255,0.19555199146270752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,255,0.19037334124247232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,255,0.42814401785532635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,255,0.17457600434621176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,255,0.16204800208409628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,255,0.2661919991175334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,255,0.2074133356412252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,255,0.15524799625078836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,255,0.15184533596038818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,511,0.51528000831604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,255,0.1509119967619578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,511,0.38419731458028156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,511,0.3001333276430766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,511,0.3112373352050781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,511,0.3347626527150472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,511,0.2884959975878398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,511,0.29443732897440594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,511,0.5101653337478638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,511,0.2890346646308899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,511,0.3219253420829773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,511,0.26311467091242474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,511,0.23149333397547403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,511,0.21708265940348306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,511,0.21174399058024088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,511,0.2071146567662557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,511,0.20806399981180826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,1023,0.5821919838587443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,1023,0.7409706910451254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,1023,0.5297600030899048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,1023,0.5068159898122152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,1023,0.4945919911066691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,1023,0.4840799967447917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,1023,0.4808266560236613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,1023,0.6960693200429281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,1023,0.43241600195566815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,1023,0.47889065742492676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,1023,0.3719093402226766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,1023,0.3237280050913493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,1023,0.31772265831629437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,1023,0.3416106700897217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,1023,0.3154346744219462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,1023,0.31752532720565796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,2047,1.0022719701131184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,2047,1.2013226350148518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,2047,0.9534613291422526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,2047,0.9146347045898438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,2047,0.8885813554128011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,2047,0.873802661895752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,2047,0.8775200049082438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,2047,0.8663040002187093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,2047,1.0631199677785237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,2047,0.6589226722717285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,2047,0.5945333242416382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,2047,0.5603573322296143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,2047,0.5337333281834921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,2047,0.5242613156636556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,2047,0.5150133371353149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,2047,0.5155093272527059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,4095,1.7994240125020344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,4095,2.102234681447347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,4095,1.7439093589782715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,4095,1.6467679341634114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,4095,1.663413365681966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,4095,1.6833066940307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,4095,1.6373227437337239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,4095,1.6313652992248535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,4095,1.8077599207560222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,4095,1.0977173646291096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,4095,1.0269760290781658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,4095,0.9537493387858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,4095,0.9804320335388184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,4095,0.9397013187408447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,4095,0.9303733507792155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,4095,0.9272533257802328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,8191,3.36187744140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,8191,3.855797449747721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,8191,3.3067893981933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,8191,3.226917266845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,8191,3.192789395650228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,8191,3.181722640991211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,8191,3.1676108042399087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,8191,3.173669179280599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,8191,3.299663861592611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,8191,1.942896048227946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,8191,1.859674612681071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,8191,1.8031147321065266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,8191,1.7694133122762044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,8191,1.7576266924540203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,8191,1.7398667335510254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,8191,1.7513866424560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,64,1024,1,2,16383,6.432714462280273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,128,1024,1,1,16383,7.319530487060547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,32,1024,1,4,16383,6.415701548258464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,8,1024,1,16,16383,6.260794957478841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,1,1024,1,128,16383,6.227925618489583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,2,1024,1,64,16383,6.237887700398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,16,1024,1,8,16383,6.2864424387613935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,float16,4,1024,1,32,16383,6.2507680257161455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,64,1024,1,2,16383,3.6017227172851562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,128,1024,1,1,16383,6.164693196614583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,32,1024,1,4,16383,3.504272143046061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,16,1024,1,8,16383,3.438074747721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,8,1024,1,16,16383,3.401365280151367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,4,1024,1,32,16383,3.39305051167806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,2,1024,1,64,16383,3.381711959838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,mla_generation,default,float16,fp8,1,1024,1,128,16383,3.3773120244344077
