framework,version,device,op_name,kernel_source,bmm_dtype,num_tokens,num_heads,latency
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1,128,0.005776000022888183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,128,0.008921600133180618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,64,0.008195199817419053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1,32,0.004800000041723251
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,32,0.008483199775218964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1,16,0.005020799860358238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1,64,0.005110400170087815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1,8,0.004185599833726883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,16,0.008403199911117553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1,4,0.004540799930691719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,4,0.00856959968805313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1,2,0.004454400017857551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,2,0.008323200047016144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1,1,0.004470400139689445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2,128,0.0060095999389886854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,8,0.007823999971151352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2,64,0.0050687998533248905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2,8,0.004915200173854828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,128,0.009414400160312652
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2,4,0.0051968000829219815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,4,0.008604799956083297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2,2,0.005244800075888634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2,32,0.004774399846792221
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,2,0.00843840017914772
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,1,0.00830719992518425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2,1,0.004438399896025657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4,128,0.005987200140953064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2,16,0.004390399903059006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4,64,0.005129599943757057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4,32,0.005049600079655647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,32,0.00883840024471283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4,16,0.0045855998992919925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,16,0.008582399785518646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4,8,0.004550400003790855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,8,0.008428800106048583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4,4,0.004614400118589402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,4,0.008579199761152267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4,2,0.004560000076889992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,2,0.00856959968805313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4,1,0.004428799822926521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,1,0.00928959995508194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8,128,0.005948799848556519
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,128,0.01071999967098236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,16,0.008499199897050858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8,64,0.0047807998955249785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,64,0.009244800359010697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8,32,0.00490880012512207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,32,0.009097599983215332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8,16,0.004620800167322159
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8,8,0.004575999826192856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,16,0.0088128000497818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,8,0.008585599809885025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8,4,0.0044895999133586885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,4,0.00859839990735054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8,2,0.004383999854326248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,2,0.008508799970149994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8,1,0.0042015999555587765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,1,0.008371199667453765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,1,0.007711999863386154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16,128,0.006099199876189232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,128,0.012486399710178375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,64,0.01010880023241043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16,32,0.005190400034189224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,32,0.009590400010347366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16,16,0.004543999955058098
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,16,0.009116800129413604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16,8,0.004419200122356415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,8,0.008524800091981888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16,4,0.0042720001190900804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16,64,0.00485760010778904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16,2,0.004236799851059914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,4,0.008473599702119828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16,1,0.004268800094723702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,2,0.007974400371313094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,1,0.008422400057315826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,32,128,0.006358399987220764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,32,64,0.005187200009822845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,128,0.015705600380897522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,64,0.011718399822711945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,32,32,0.004681599885225296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,32,16,0.0045311998575925825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,32,0.010073599964380264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,16,0.009411200135946273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,8,0.007999999821186066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,32,8,0.0045056000351905824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,64,0.008451200276613235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,4,0.008703999966382981
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,32,2,0.004383999854326248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,32,0.00827839970588684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,8,0.00886079967021942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,32,4,0.004447999969124794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,2,0.007980799674987793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,32,1,0.005296000093221664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,1,0.008684799820184708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,128,0.01924159973859787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,48,64,0.005337600037455559
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,64,0.008671999722719193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,128,0.009385599941015243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,64,0.013209599256515502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,48,32,0.005004800111055374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,32,0.011443199962377549
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,48,16,0.005171199887990951
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,16,0.009939199686050415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,48,8,0.004675199836492538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,8,0.009171199798583985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,4,0.008729600161314011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,48,2,0.004825599864125252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,2,0.008934400230646133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,48,128,0.006748799979686737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,48,1,0.004441599920392036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,1,0.008736000210046769
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,64,128,0.0072672002017498015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,128,0.02234559953212738
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,64,64,0.005584000051021576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,64,0.01499200016260147
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,64,32,0.004771199822425842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,32,0.011635199934244157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,64,16,0.004649600014090538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,16,0.01037440001964569
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,64,8,0.004681599885225296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,48,4,0.005443200096487999
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,64,1,0.004454400017857551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,8,0.009440000355243682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,64,4,0.00432640016078949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,4,0.008771199733018875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,64,2,0.004383999854326248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,2,0.008985599875450135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,1,0.008425600081682205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,80,128,0.007769600301980972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,128,0.02561280131340027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,80,64,0.005900799855589866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,64,0.016982400417327882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,80,32,0.005078399926424027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,8,0.009644799679517747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,32,0.012950399518013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,80,16,0.004582399874925614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,16,0.010665600001811982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,80,8,0.0046847999095916745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,80,4,0.004441599920392036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,4,0.009033600240945816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,80,2,0.00451200008392334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,2,0.008764799684286118
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,80,1,0.006191999837756157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,1,0.008633600175380706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,96,128,0.008054400235414505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,128,0.029366400837898255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,96,64,0.00589120015501976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,64,0.018768000602722167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,96,32,0.005152000114321709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,32,0.013945600390434265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,96,16,0.004960000142455101
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,16,0.010937599837779999
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,96,8,0.004639999940991402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,8,0.009811200201511383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,96,4,0.0047391999512910845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,4,0.009151999652385712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,96,2,0.005980800092220307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,2,0.0088128000497818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,96,1,0.004492799937725067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,1,0.008534400165081025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,128,128,0.009353599697351455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,128,0.03869760036468506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,128,64,0.0063231997191905975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,64,0.02215999960899353
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,128,32,0.005849599838256836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,32,0.015372799336910247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,128,16,0.004787199944257736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,16,0.011657600104808808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,128,8,0.00488319993019104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,8,0.010294400155544281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,128,4,0.004764800146222115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,4,0.009401600062847137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,128,0.04741120040416717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,128,2,0.004329600185155868
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,2,0.009030400216579438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,128,1,0.0044895999133586885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,1,0.008796799927949905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,160,128,0.01656319946050644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,160,64,0.006911999732255936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,160,32,0.0056223999708890915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,64,0.025337600708007814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,32,0.016867199540138246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,160,16,0.005673599988222122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,16,0.012969599664211273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,160,8,0.00488319993019104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,8,0.010566399991512298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,160,4,0.0046720001846551895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,4,0.009487999975681305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,160,2,0.0047775998711586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,2,0.009177599847316743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,160,1,0.00498879998922348
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,1,0.00851840004324913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,192,128,0.02186879962682724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,128,0.05481280088424682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,16,0.013395200669765472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,192,64,0.007302399724721909
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,64,0.028700798749923706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,192,32,0.006384000182151794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,32,0.018287999927997588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,192,16,0.00520000010728836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,192,8,0.005008000135421753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,8,0.01127680018544197
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,1,0.00886079967021942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,192,4,0.004560000076889992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,4,0.009734400361776353
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,192,2,0.004623999819159508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,2,0.009424000233411788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,192,1,0.004358400031924248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,256,128,0.025017601251602174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,128,0.07029119729995728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,256,64,0.007884799689054488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,64,0.03663040101528168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,256,32,0.006143999844789505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,32,0.022284799814224245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,256,16,0.004992000013589859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,16,0.015039999783039094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,256,8,0.004790399968624115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,8,0.011635199934244157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,256,4,0.005132799968123436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,4,0.010230399668216705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,320,128,0.028828799724578857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,256,2,0.006310400366783142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,2,0.009283199906349182
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,256,1,0.004377600178122521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,1,0.008991999924182892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,128,0.08679680228233337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,320,64,0.009363199770450591
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,64,0.0455839991569519
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,320,32,0.006889600306749344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,32,0.025225600600242613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,320,16,0.005056000128388405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,16,0.016892799735069276
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,320,8,0.005353600159287453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,320,1,0.004502400010824204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,8,0.013011200726032257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,320,4,0.00453759990632534
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,4,0.010502400249242783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,320,2,0.0045471999794244765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,2,0.00939520001411438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,1,0.009164799749851228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,384,128,0.03313600122928619
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,128,0.10340160131454468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,384,64,0.01326719969511032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,64,0.05370879769325256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,384,32,0.0070271998643875126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,32,0.02858240008354187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,384,16,0.00549440011382103
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,384,8,0.0050592001527547835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,16,0.018646399676799773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,8,0.013663999736309052
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,384,4,0.0047136001288890835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,4,0.011055999994277954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,384,2,0.0044895999133586885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,2,0.00957759991288185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,384,1,0.004761600121855736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,1,0.009055999666452407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,512,128,0.040870401263237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,128,0.13425920009613038
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,512,64,0.022918400168418885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,64,0.0695904016494751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,512,32,0.00788159966468811
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,32,0.03603839874267578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,512,16,0.005929600074887275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,16,0.021878400444984437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,512,8,0.005104000121355057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,8,0.015244799852371215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,512,4,0.004822399839758873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,4,0.011939200013875962
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,512,2,0.0047775998711586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,2,0.010288000106811523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,512,1,0.005603199824690819
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,1,0.009308800101280212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,768,128,0.05661439895629883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,128,0.1961567997932434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,768,64,0.030457600951194763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,64,0.10168960094451904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,768,32,0.009827200323343277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,32,0.053276801109313966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,768,16,0.007372800260782242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,16,0.02863680124282837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,768,8,0.005459199845790863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,8,0.018512000143527985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,768,4,0.004899200052022934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,4,0.013750399649143218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,768,2,0.004729599878191948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,2,0.010886400192975997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,768,1,0.004915200173854828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,1,0.009804800152778625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1024,128,0.07238079905509949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1024,64,0.03809599876403809
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,128,0.25829439163208007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,64,0.13257600069046022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1024,32,0.020083199441432952
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,32,0.0686847984790802
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1024,16,0.008224000036716462
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,16,0.03588800132274628
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1024,8,0.005817599967122078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,8,0.021660800278186797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1024,4,0.005110400170087815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,4,0.01520639955997467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1024,2,0.004748800024390221
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,2,0.012035199999809265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1024,1,0.005299200117588043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,1,0.010073599964380264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1536,128,0.10286719799041748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1536,64,0.05494719743728638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,128,0.3868191957473755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1536,32,0.02905279994010925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,64,0.1965376019477844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,8,0.028457599878311157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1536,16,0.009839999675750732
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,32,0.101363205909729
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,16,0.05267519950866699
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1536,8,0.007382400333881378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1536,4,0.005423999950289726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,4,0.018569600582122803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1536,2,0.00493439994752407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,2,0.013708800077438354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,1536,1,0.00483199991285801
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,1,0.010809600353240967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2048,128,0.13246079683303832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2048,64,0.0694815993309021
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,128,0.5204800128936767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,64,0.25723841190338137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2048,32,0.03696640133857727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,32,0.1314687967300415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2048,16,0.01857919991016388
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,16,0.06855679750442505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2048,8,0.00809279978275299
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,8,0.036355200409889224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2048,4,0.005667199939489364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,4,0.021913599967956544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,3072,128,0.19253120422363282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2048,2,0.005206400156021118
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,2,0.015263999998569488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,2048,1,0.004879999905824661
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,1,0.011791999638080596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,3072,64,0.09799360036849976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,3072,32,0.0531391978263855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,128,0.8113375663757324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,64,0.3850719928741455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,32,0.1955839991569519
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,3072,4,0.007212799787521362
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,4,0.028825598955154418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,3072,2,0.005379199981689453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,2,0.018367999792099
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,3072,16,0.028252801299095152
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,1,0.013775999844074249
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,3072,8,0.010003200173377991
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,16,0.10066239833831787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,8,0.0531391978263855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,3072,1,0.005004800111055374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4096,128,0.251145601272583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4096,64,0.1280992031097412
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4096,32,0.0679967999458313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,64,0.5276639938354493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,128,1.1056096076965332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,32,0.25699520111083984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4096,16,0.036259201169013974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,16,0.1316383957862854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4096,8,0.017375999689102174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,8,0.06855999827384948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4096,4,0.007983999699354172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,4,0.03605439960956573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4096,2,0.005798399820923806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,2,0.021807999908924104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,4096,1,0.005046400055289268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,1,0.015318399667739869
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,6144,128,0.3699199914932251
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,6144,64,0.18967360258102417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,6144,32,0.09818559885025024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,64,0.812172794342041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,6144,16,0.052534401416778564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,32,0.38386878967285154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,4,0.05299519896507263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,6144,2,0.007302399724721909
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,128,1.6307615280151366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,16,0.19410560131072999
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,6144,8,0.0312032014131546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,8,0.10009599924087524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,6144,4,0.009993600100278855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,2,0.02845120131969452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,6144,1,0.005420799925923347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,1,0.018566399812698364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8192,64,0.24534718990325927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8192,128,0.5487199783325195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,16,0.256822395324707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8192,32,0.1260480046272278
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8192,16,0.06707839965820313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,32,0.5192448139190674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,64,1.1034239768981933
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8192,8,0.0400191992521286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,128,2.2688671112060548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8192,4,0.017420800030231477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,8,0.1345471978187561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,4,0.06836479902267456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8192,2,0.00809599980711937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,2,0.03566400110721588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,8192,1,0.005776000022888183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,1,0.021695999801158904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,12288,64,0.360099196434021
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,12288,128,0.7867551803588867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,12288,32,0.18538880348205566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,12288,16,0.09692479968070984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,32,0.809500789642334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,64,1.6916831970214843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,12288,8,0.05803840160369873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,4,0.09956160187721252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,16,0.3921152114868164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,12288,1,0.007187200337648391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,8,0.19547200202941895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,12288,4,0.027619200944900512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,128,3.2757919311523436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,12288,2,0.009996800124645234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,2,0.05268160104751587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,1,0.028646400570869444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16384,128,0.9687840461730957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16384,64,0.512608003616333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16384,32,0.24332799911499023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16384,16,0.12612160444259643
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16384,8,0.07530879974365234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,32,1.1116479873657226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,16,0.5273312091827392
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16384,4,0.03575679957866669
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,8,0.2557120084762573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,64,2.2481664657592773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,4,0.13097280263900757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16384,2,0.015945599973201753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,2,0.06906560063362122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,16384,1,0.007900799810886382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,1,0.03591679930686951
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,128,4.542086410522461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,20480,64,0.5933824062347413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,20480,32,0.30014400482177733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,20480,16,0.1552448034286499
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,20480,128,1.2428799629211427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,20480,8,0.09290239810943604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,8,0.3185983896255493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,20480,2,0.023263999819755556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,16,0.6665503978729248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,20480,4,0.04557119905948639
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,4,0.16458879709243773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,32,1.4007743835449218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,float16,20480,1,0.008854400366544724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,2,0.08394240140914917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,1,0.04586879909038544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,64,2.835708808898926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,128,5.712998580932617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1,128,0.006025600060820579
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1,64,0.005014400184154511
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,128,0.00920960009098053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1,32,0.0042559999972581865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,32,0.00852160006761551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1,16,0.004153599962592125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,16,0.008259200304746629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1,8,0.003999999910593033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,8,0.0080704003572464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,64,0.008659200370311737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1,4,0.004025600105524063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,4,0.007571200281381607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1,2,0.00398080013692379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1,1,0.004112000018358231
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,2,0.008265600353479386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,1,0.007968000322580337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2,128,0.006332799792289734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2,64,0.005119999870657921
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,128,0.009891200065612792
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,64,0.009347199648618697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2,32,0.004681599885225296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,32,0.008665599673986436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2,16,0.0046431999653577805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,16,0.008428800106048583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2,8,0.004543999955058098
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,8,0.008300799876451492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2,4,0.004582399874925614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,4,0.00806720033288002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2,2,0.003942399844527244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,2,0.008345600217580795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2,1,0.00398080013692379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,1,0.008182399719953538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,128,0.01016639992594719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4,64,0.005337600037455559
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,16,0.008448000252246856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4,8,0.005129599943757057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4,4,0.003971200063824653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,4,0.008287999778985977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4,2,0.004009599983692169
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,2,0.008470399677753449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4,1,0.003920000046491623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,1,0.008144000172615051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8,128,0.006076800078153611
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,128,0.010873600095510482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8,64,0.004940799996256828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,64,0.00986879989504814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8,32,0.0043519999831914905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,32,0.00896959975361824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8,16,0.004150399938225746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,16,0.008540800213813782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8,8,0.0040352001786232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,8,0.008416000008583068
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8,4,0.0038943998515605927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,4,0.008294399827718735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8,2,0.00398080013692379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,2,0.008371199667453765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8,1,0.003990399837493897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,1,0.008172799646854401
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,16,128,0.006393600255250931
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,16,64,0.005087999999523163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4,128,0.006108799949288368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,16,32,0.004588799923658371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4,16,0.004467200115323066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4,32,0.0044064000248909
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,16,16,0.004508800059556961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,16,8,0.004742399975657463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,128,0.01329279989004135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,64,0.010384000092744827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,16,4,0.0051807999610900875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,4,0.008271999657154083
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,16,2,0.005289600044488907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,2,0.008214399963617326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,16,1,0.004912000149488449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,1,0.008083199709653854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,32,128,0.006803199648857117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,128,0.01603520065546036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,32,64,0.005385600030422211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,64,0.01223360002040863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,32,32,0.005020799860358238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,32,0.010041599720716476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,32,16,0.005872000008821487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,16,0.009251199662685394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,32,8,0.005030399933457374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,8,0.00857279971241951
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,32,4,0.005177599936723709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,4,0.008470399677753449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,32,2,0.004931199923157692
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,2,0.008300799876451492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,32,1,0.0050432000309228895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,1,0.008172799646854401
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,48,128,0.007302399724721909
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,128,0.019523200392723084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,48,64,0.005798399820923806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,64,0.01417279988527298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,48,32,0.005040000006556511
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,32,0.010896000266075134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,48,16,0.004979199916124344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,16,0.00950080007314682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,48,8,0.00594559982419014
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,8,0.00873280018568039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,48,4,0.005027199909090996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,4,0.008665599673986436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,48,2,0.005167999863624572
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,2,0.00827839970588684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,48,1,0.0050432000309228895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,1,0.008243200182914735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,64,128,0.007846400141716003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,128,0.023395200073719025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,64,64,0.006108799949288368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,64,0.01648000031709671
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,64,32,0.005094400048255921
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,32,0.011852800101041793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,8,0.007680000364780426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,64,16,0.004985599964857102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,32,0.008262400329113007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,64,0.008668799698352814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,64,8,0.00514880008995533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,8,0.008604799956083297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,16,0.009510400146245957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,4,0.00878399983048439
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,2,0.00817599967122078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,64,1,0.004927999898791313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,1,0.00825600028038025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,80,128,0.0084927998483181
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,80,64,0.0066143997013568875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,128,0.026931199431419372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,64,0.017846399545669557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,32,0.013254399597644805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,80,16,0.005071999877691269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,16,0.008275199681520462
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,8,0.008550400286912918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,32,0.008931200206279754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,16,0.010531199723482132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,80,8,0.004095999896526337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,64,4,0.004227200150489807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,8,0.009379199892282485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,64,2,0.003958399966359138
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,80,4,0.004355200007557869
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,4,0.008963199704885483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,80,2,0.004009599983692169
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,2,0.00854720026254654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,80,1,0.004105599969625473
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,1,0.008195199817419053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,96,128,0.009027200192213059
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,32,0.013891200721263885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,128,0.03049600124359131
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,80,32,0.005152000114321709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,96,64,0.006825599819421768
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,64,0.019654400646686554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,96,32,0.0052928000688552855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,96,16,0.004630399867892265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,16,0.011072000116109848
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,2,0.008588799834251403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,96,8,0.0042559999972581865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,8,0.009609600156545639
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,128,128,0.011139199882745743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,96,4,0.004105599969625473
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,4,0.008928000181913375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,96,2,0.004912000149488449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,96,1,0.004745600000023842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,1,0.00830719992518425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,128,0.03802880048751831
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,128,64,0.007779199630022049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,64,0.023254400491714476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,128,32,0.005459199845790863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,32,0.01565759927034378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,128,16,0.004649600014090538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,16,0.011648000031709672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,128,8,0.004377600178122521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,8,0.00981760025024414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,128,4,0.004348799958825112
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,4,0.008937600255012512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,128,0.04649600088596344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,128,2,0.00398080013692379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,2,0.008755200356245042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,128,1,0.004054399952292442
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,1,0.008303999900817871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,160,128,0.016201600432395935
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,160,64,0.008495999872684479
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,64,0.02690559923648834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,160,32,0.0061055999249219894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,32,0.01746239960193634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,160,16,0.005270399898290634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,16,0.013017599284648896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,160,8,0.004377600178122521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,8,0.01026879996061325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,160,4,0.004278400167822838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,4,0.00928959995508194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,160,2,0.004243199899792671
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,2,0.00868159979581833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,160,1,0.005315199866890908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,1,0.008617600053548813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,192,128,0.020633600652217865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,128,0.056892800331115725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,192,64,0.008841600269079208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,64,0.030457600951194763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,192,32,0.006547199934720993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,32,0.01910399943590164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,192,16,0.00520000010728836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,16,0.013817599415779114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,192,8,0.0043935999274253845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,8,0.010627199709415436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,192,4,0.004342399910092354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,4,0.009462399780750275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,192,2,0.0044096000492572784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,2,0.00873280018568039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,192,1,0.005708799883723259
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,1,0.00841279998421669
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,256,128,0.026553601026535034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,128,0.07195519804954528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,256,64,0.00987199991941452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,64,0.03738240003585815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,256,32,0.00721919983625412
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,32,0.02295999974012375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,256,16,0.005344000086188317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,16,0.01555519998073578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,256,8,0.004668800160288811
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,8,0.01180799975991249
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,256,4,0.004566400125622749
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,4,0.009753599762916565
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,256,2,0.005238400027155876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,2,0.009011200070381165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,256,1,0.0039680000394582745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,1,0.008508799970149994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,320,128,0.02971520125865936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,32,0.026342400908470155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,128,0.08698559999465942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,320,64,0.012700800597667695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,64,0.04710719883441925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,8,0.012835200130939483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,320,32,0.0073183998465538025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,320,16,0.005849599838256836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,16,0.017334400117397307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,320,8,0.004700800031423568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,320,4,0.004342399910092354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,4,0.010527999699115753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,320,2,0.0043136000633239744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,2,0.009001599997282029
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,320,1,0.004348799958825112
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,1,0.0088639996945858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,384,128,0.03426879942417145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,128,0.10262399911880493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,384,64,0.016422399878501893
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,64,0.05611199736595154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,384,32,0.008489599823951722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,32,0.03003840148448944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,384,16,0.006294400244951248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,16,0.01945600062608719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,384,8,0.004924799874424934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,8,0.013926400244235993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,384,4,0.005177599936723709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,4,0.010831999778747558
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,384,2,0.004300799965858459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,2,0.009516800194978714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,384,1,0.004358400031924248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,1,0.008799999952316284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,512,128,0.04219520092010498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,128,0.13285759687423707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,512,64,0.022614400088787078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,64,0.07120959758758545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,512,32,0.00986879989504814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,32,0.037555199861526486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,512,16,0.007161600142717361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,16,0.022995199263095855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,512,8,0.005449600145220756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,8,0.015520000457763672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,512,4,0.005084799975156784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,4,0.011433599889278412
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,512,2,0.004419200122356415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,2,0.009705600142478944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,512,1,0.00456320010125637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,1,0.009123200178146362
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,768,128,0.05761280059814453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,128,0.19614720344543457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,768,64,0.03130879998207092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,64,0.10289280414581299
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,768,32,0.014601600170135499
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,32,0.05501440167427063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,4,0.013635200262069703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,768,16,0.008854400366544724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,16,0.03048959970474243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,768,8,0.0068000003695487974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,8,0.01905920058488846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,768,4,0.00514880008995533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,768,2,0.004447999969124794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,768,1,0.0042975999414920805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,2,0.010751999914646149
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,1,0.009443199634552002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1024,128,0.07269759774208069
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1024,64,0.03979839980602264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,16,0.037539198994636536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,128,0.2589695930480957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1024,32,0.02248319983482361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,64,0.13237440586090088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,32,0.07086079716682434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1024,16,0.009948799759149552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1024,8,0.007158400118350982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,8,0.022678400576114654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1024,4,0.005488000065088272
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,4,0.015609599649906158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1024,2,0.004550400003790855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,2,0.011580800265073776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1024,1,0.004422400146722794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,1,0.009910400211811065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1536,32,0.030195200443267824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1536,128,0.1043552041053772
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1536,64,0.05451200008392334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,128,0.3852319955825806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,64,0.19568639993667603
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,32,0.10246720314025878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1536,16,0.014300799369812012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,16,0.05537599921226501
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1536,8,0.008544000238180161
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,8,0.030399999022483824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1536,4,0.006198399886488915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,4,0.01908479928970337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1536,2,0.0050016000866889955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,2,0.013648000359535218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,1536,1,0.005836800113320351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,1,0.010652799904346467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2048,128,0.13415999412536622
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2048,64,0.07059199810028076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,16,0.07030400037765502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,128,0.5142047882080079
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,64,0.2580575942993164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2048,32,0.038236799836158755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,32,0.13230400085449218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2048,16,0.020377600193023683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2048,8,0.009916800260543823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,8,0.037510401010513304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2048,4,0.007129599899053573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,4,0.022700800001621245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2048,2,0.005395200103521347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,2,0.015564799308776855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,2048,1,0.004659200087189674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,1,0.011657600104808808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,3072,128,0.19530880451202393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,32,0.1949023962020874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,3072,64,0.10183360576629638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,3072,8,0.014188799262046813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,128,0.7721471786499023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,3072,32,0.053887999057769774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,64,0.3832704067230225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,3072,16,0.029868799448013305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,16,0.101801598072052
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,8,0.055340802669525145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,3072,4,0.008595199882984161
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,4,0.030057600140571593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,3072,2,0.006307200342416763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,2,0.019100800156593323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,3072,1,0.0049632001668214795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,1,0.013631999492645264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4096,128,0.25661120414733884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4096,64,0.13243199586868287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4096,32,0.06946240067481994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,64,0.5118591785430908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,128,1.036796760559082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4096,16,0.03769280016422272
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,32,0.2568896055221558
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,16,0.13176000118255615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4096,8,0.019340799748897554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,8,0.0704479992389679
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4096,4,0.010143999755382539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,4,0.03761279881000519
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4096,2,0.007340800017118454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,2,0.022771200537681578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,4096,1,0.0052767999470233916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,1,0.015523199737071992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,6144,128,0.3811072111129761
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,6144,64,0.19419840574264527
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,6144,32,0.09965760111808777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,64,0.7707712173461914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,6144,16,0.053235197067260744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,32,0.3821727991104126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,128,1.5705663681030273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,16,0.1943168044090271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,6144,8,0.029555198550224305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,8,0.10234240293502808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,6144,4,0.014703999459743499
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,4,0.05506560206413269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,6144,2,0.00857279971241951
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,2,0.030195200443267824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,6144,1,0.006195199862122536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,1,0.019020800292491914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8192,128,0.5031231880187989
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8192,64,0.2560703992843628
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8192,32,0.13136639595031738
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8192,16,0.06988160014152527
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,64,1.0347264289855957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,32,0.5094816207885742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,16,0.25695679187774656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,128,2.093107223510742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8192,8,0.04217599928379059
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,8,0.1325536012649536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8192,4,0.019724799692630766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,4,0.07067840099334717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8192,2,0.010092800110578537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,2,0.03776960074901581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,float16,8192,1,0.007286400347948074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,1,0.023228800296783446
