framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,4,1,0,0.10720000267028809
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,8,1,0,0.10814399719238281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,16,1,0,0.10785599946975707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,32,1,0,0.11666239500045776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,64,1,0,0.11385920047760009
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,128,1,0,0.11326080560684204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,256,1,0,0.11292799711227416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,512,1,0,0.11348799467086793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,1024,1,0,0.1137503981590271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,2048,1,0,0.11559679508209228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,4096,1,0,0.11886080503463745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,8192,1,0,0.12383359670639038
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1,16384,1,0,0.13392000198364257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,4,1,0,0.10876799821853637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,8,1,0,0.10901119709014892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,16,1,0,0.10932480096817017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,32,1,0,0.11965440511703491
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,64,1,0,0.11594239473342896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,128,1,0,0.11466560363769532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,256,1,0,0.11438720226287842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,512,1,0,0.11477119922637939
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,1024,1,0,0.11638720035552978
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,2048,1,0,0.11935679912567139
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,4096,1,0,0.12095040082931519
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,8192,1,0,0.1275712013244629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,2,16384,1,0,0.13538559675216674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,4,1,0,0.1100160002708435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,8,1,0,0.11015039682388306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,16,1,0,0.1109887957572937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,32,1,0,0.12157119512557983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,64,1,0,0.11864320039749146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,128,1,0,0.11789120435714721
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,256,1,0,0.1187391996383667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,512,1,0,0.11978880167007447
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,1024,1,0,0.12190079689025879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,2048,1,0,0.12367680072784423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,4096,1,0,0.1322335958480835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,8192,1,0,0.13953919410705568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,4,16384,1,0,0.15429120063781737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,4,1,0,0.1088096022605896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,8,1,0,0.1091264009475708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,16,1,0,0.10915520191192626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,32,1,0,0.12037760019302368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,64,1,0,0.11789439916610718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,128,1,0,0.11803200244903564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,256,1,0,0.11929600238800049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,512,1,0,0.12096320390701294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,1024,1,0,0.12205760478973389
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,2048,1,0,0.13004159927368164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,4096,1,0,0.13751039505004883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,8192,1,0,0.15368959903717042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,8,16384,1,0,0.18608959913253784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,4,1,0,0.11018559932708741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,8,1,0,0.11119040250778198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,16,1,0,0.11068480014801026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,32,1,0,0.12360639572143554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,64,1,0,0.1229632019996643
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,128,1,0,0.12600640058517457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,256,1,0,0.12660479545593262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,512,1,0,0.1266335964202881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,1024,1,0,0.13296960592269896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,2048,1,0,0.14264639616012573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,4096,1,0,0.15802559852600098
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,8192,1,0,0.1890015959739685
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,16,16384,1,0,0.2526335954666138
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,4,1,0,0.11219840049743653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,8,1,0,0.11211199760437011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,16,1,0,0.11191680431365966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,32,1,0,0.11215360164642334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,64,1,0,0.114028799533844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,128,1,0,0.11597119569778443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,256,1,0,0.13171839714050293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,512,1,0,0.1413632035255432
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,1024,1,0,0.1501312017440796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,2048,1,0,0.16681280136108398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,4096,1,0,0.19735679626464844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,8192,1,0,0.2590719938278198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,32,16384,1,0,0.37861759662628175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,4,1,0,0.11767040491104126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,8,1,0,0.11784640550613404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,16,1,0,0.11805440187454223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,32,1,0,0.11778240203857422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,64,1,0,0.11952639818191528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,128,1,0,0.121670401096344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,256,1,0,0.1528480052947998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,512,1,0,0.1686624050140381
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,1024,1,0,0.187283194065094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,2048,1,0,0.21968960762023926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,4096,1,0,0.2803679943084717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,8192,1,0,0.4009280204772949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,64,16384,1,0,0.6673567771911622
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,4,1,0,0.13566720485687256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,8,1,0,0.1355520009994507
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,16,1,0,0.13572160005569459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,32,1,0,0.13715200424194335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,64,1,0,0.14031039476394652
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,128,1,0,0.14382719993591309
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,256,1,0,0.15410879850387574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,512,1,0,0.17261439561843872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,1024,1,0,0.20472640991210939
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,2048,1,0,0.26691200733184817
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,4096,1,0,0.3890079975128174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,8192,1,0,0.62991361618042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,128,16384,1,0,1.1147135734558105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,256,4,1,0,0.18320640325546264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,256,8,1,0,0.18272320032119752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,256,16,1,0,0.18254079818725585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,256,32,1,0,0.18522239923477174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,256,64,1,0,0.1914240002632141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,256,128,1,0,0.200598406791687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,256,256,1,0,0.21691839694976806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,256,512,1,0,0.2527904033660889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,256,1024,1,0,0.3175872087478638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,256,2048,1,0,0.43968639373779295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,256,4096,1,0,0.6816959857940674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,256,8192,1,0,1.1633119583129883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,512,4,1,0,0.314847993850708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,512,8,1,0,0.3142560005187988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,512,16,1,0,0.31199040412902834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,512,32,1,0,0.3144351959228516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,512,64,1,0,0.32556159496307374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,512,128,1,0,0.3449824094772339
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,512,256,1,0,0.3779871940612793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,512,512,1,0,0.44805121421813965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,512,1024,1,0,0.5763455867767334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,512,2048,1,0,0.8217375755310059
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,512,4096,1,0,1.3041472434997559
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1024,4,1,0,0.5716832160949707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1024,8,1,0,0.5714975833892822
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1024,16,1,0,0.5723936080932617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1024,32,1,0,0.5796703815460205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1024,64,1,0,0.6003424167633057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1024,128,1,0,0.6296319961547852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1024,256,1,0,0.7022047996520996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1024,512,1,0,0.8377535820007325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1024,1024,1,0,1.0960607528686523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,128,1024,2048,1,0,1.5901663780212403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,4,1,0,0.0740447998046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,8,1,0,0.07387199997901917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,16,1,0,0.07410879731178284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,32,1,0,0.08314239978790283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,64,1,0,0.08090559840202331
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,128,1,0,0.07956159710884095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,256,1,0,0.07965760231018067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,512,1,0,0.07966079711914062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,1024,1,0,0.0838047981262207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,2048,1,0,0.083651202917099
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,4096,1,0,0.08603839874267578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,8192,1,0,0.0896992027759552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1,16384,1,0,0.09726079702377319
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,4,1,0,0.07398719787597656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,8,1,0,0.07449920177459717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,16,1,0,0.07438079714775085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,32,1,0,0.08318399786949157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,64,1,0,0.08122559785842895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,128,1,0,0.08036159873008727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,256,1,0,0.07943040132522583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,512,1,0,0.07984960079193115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,1024,1,0,0.08007360100746155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,2048,1,0,0.08195840120315552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,4096,1,0,0.08580160140991211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,8192,1,0,0.09115840196609497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,2,16384,1,0,0.10504640340805053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,4,1,0,0.07502400279045104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,8,1,0,0.0747871994972229
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,16,1,0,0.07496320009231568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,32,1,0,0.0852512001991272
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,64,1,0,0.08238080143928528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,128,1,0,0.08141440153121948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,256,1,0,0.08110079765319825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,512,1,0,0.08163840174674988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,1024,1,0,0.08340799808502197
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,2048,1,0,0.08633279800415039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,4096,1,0,0.0883679986000061
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,8192,1,0,0.0982047975063324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,4,16384,1,0,0.11164480447769165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,4,1,0,0.07376319766044617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,8,1,0,0.07323200106620789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,16,1,0,0.07434239983558655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,32,1,0,0.08495360016822814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,64,1,0,0.08164479732513427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,128,1,0,0.08146880269050598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,256,1,0,0.08182399868965148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,512,1,0,0.08415359854698182
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,1024,1,0,0.08595839738845826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,2048,1,0,0.08909760117530822
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,4096,1,0,0.10066560506820679
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,8192,1,0,0.11234240531921387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,8,16384,1,0,0.13651520013809204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,4,1,0,0.07376639842987061
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,8,1,0,0.07401919960975648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,16,1,0,0.07427840232849121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,32,1,0,0.08566719889640809
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,64,1,0,0.08223360180854797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,128,1,0,0.08332160115242004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,256,1,0,0.08416960239410401
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,512,1,0,0.0864192008972168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,1024,1,0,0.0892416000366211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,2048,1,0,0.09913600087165833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,4096,1,0,0.11305279731750488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,8192,1,0,0.1373855948448181
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,16,16384,1,0,0.18611840009689332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,4,1,0,0.07485759854316712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,8,1,0,0.0753279983997345
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,16,1,0,0.07539520263671876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,32,1,0,0.08786879777908325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,64,1,0,0.08740800023078918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,128,1,0,0.08950080275535584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,256,1,0,0.09158400297164918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,512,1,0,0.09231359958648681
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,1024,1,0,0.10631999969482422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,2048,1,0,0.11882879734039306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,4096,1,0,0.1435263991355896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,8192,1,0,0.19133119583129882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,32,16384,1,0,0.28657920360565187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,4,1,0,0.07820799946784973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,8,1,0,0.0782912015914917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,16,1,0,0.07871999740600585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,32,1,0,0.07846400141716003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,64,1,0,0.08044480085372925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,128,1,0,0.0821951985359192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,256,1,0,0.09861119985580444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,512,1,0,0.11074880361557007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,1024,1,0,0.12705600261688232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,2048,1,0,0.15321600437164307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,4096,1,0,0.20215039253234862
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,8192,1,0,0.2993119955062866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,64,16384,1,0,0.4909664154052734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,4,1,0,0.08656319975852966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,8,1,0,0.08637760281562805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,16,1,0,0.08642879724502564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,32,1,0,0.08736000061035157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,64,1,0,0.08874880075454712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,128,1,0,0.09051200151443481
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,256,1,0,0.1256160020828247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,512,1,0,0.14905279874801636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,1024,1,0,0.17322560548782348
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,2048,1,0,0.22190399169921876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,4096,1,0,0.31902079582214354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,8192,1,0,0.5133024215698242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,128,16384,1,0,0.9168959617614746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,256,4,1,0,0.1068511962890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,256,8,1,0,0.10667840242385865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,256,16,1,0,0.10692479610443115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,256,32,1,0,0.10910719633102417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,256,64,1,0,0.11363519430160522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,256,128,1,0,0.11957119703292847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,256,256,1,0,0.13587839603424073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,256,512,1,0,0.16285760402679444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,256,1024,1,0,0.21154239177703857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,256,2048,1,0,0.30874879360198976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,256,4096,1,0,0.501964807510376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,256,8192,1,0,0.8874239921569824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,512,4,1,0,0.17475520372390746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,512,8,1,0,0.1750656008720398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,512,16,1,0,0.17574399709701538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,512,32,1,0,0.17885440587997437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,512,64,1,0,0.1857311964035034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,512,128,1,0,0.19998079538345337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,512,256,1,0,0.2276479959487915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,512,512,1,0,0.27998719215393064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,512,1024,1,0,0.3773567914962769
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,512,2048,1,0,0.5714431762695312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,512,4096,1,0,0.9566176414489747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1024,4,1,0,0.3121567964553833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1024,8,1,0,0.3113023996353149
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1024,16,1,0,0.3109312057495117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1024,32,1,0,0.3156896114349365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1024,64,1,0,0.32858240604400635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1024,128,1,0,0.3554719924926758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1024,256,1,0,0.40797119140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1024,512,1,0,0.512828779220581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1024,1024,1,0,0.7076096057891845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,64,1024,2048,1,0,1.095193576812744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,4,1,0,0.058089601993560794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,8,1,0,0.05800960063934326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,16,1,0,0.057974398136138916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,32,1,0,0.06217600107192993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,64,1,0,0.061654400825500486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,128,1,0,0.06105599999427795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,256,1,0,0.0606112003326416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,512,1,0,0.06228160262107849
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,1024,1,0,0.06664959788322448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,2048,1,0,0.06714879870414733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,4096,1,0,0.06878399848937988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,8192,1,0,0.07309439778327942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1,16384,1,0,0.08093439936637878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,4,1,0,0.05648639798164368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,8,1,0,0.05629119873046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,16,1,0,0.056777602434158324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,32,1,0,0.0616703987121582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,64,1,0,0.060399997234344485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,128,1,0,0.05959360003471374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,256,1,0,0.05977920293807983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,512,1,0,0.06212159991264343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,1024,1,0,0.06208320260047913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,2048,1,0,0.064028799533844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,4096,1,0,0.06803839802742004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,8192,1,0,0.07310720086097718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,2,16384,1,0,0.08634880185127258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,4,1,0,0.05713599920272827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,8,1,0,0.057708799839019775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,16,1,0,0.05730559825897217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,32,1,0,0.06218559741973877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,64,1,0,0.06097919940948486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,128,1,0,0.06011840105056763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,256,1,0,0.060121601819992064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,512,1,0,0.06104320287704468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,1024,1,0,0.06273599863052368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,2048,1,0,0.06584960222244263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,4096,1,0,0.06980159878730774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,8192,1,0,0.07955520153045655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,4,16384,1,0,0.09255040287971497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,4,1,0,0.05553280115127564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,8,1,0,0.054425597190856934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,16,1,0,0.05586559772491455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,32,1,0,0.06113280057907104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,64,1,0,0.0594976007938385
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,128,1,0,0.05912320017814636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,256,1,0,0.059443199634552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,512,1,0,0.061887997388839724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,1024,1,0,0.06435199975967407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,2048,1,0,0.06642240285873413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,4096,1,0,0.07984960079193115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,8192,1,0,0.09423360228538513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,8,16384,1,0,0.11704319715499878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,4,1,0,0.05529599785804749
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,8,1,0,0.05614079833030701
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,16,1,0,0.056169599294662476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,32,1,0,0.062009602785110474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,64,1,0,0.06061760187149048
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,128,1,0,0.06059200167655945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,256,1,0,0.06234560012817383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,512,1,0,0.06441599726676941
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,1024,1,0,0.06753280162811279
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,2048,1,0,0.07784000039100647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,4096,1,0,0.0915232002735138
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,8192,1,0,0.1146239995956421
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,16,16384,1,0,0.1646496057510376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,4,1,0,0.05638719797134399
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,8,1,0,0.05663679838180542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,16,1,0,0.056201601028442384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,32,1,0,0.06344320178031922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,64,1,0,0.06380159854888916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,128,1,0,0.0655456006526947
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,256,1,0,0.0672864019870758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,512,1,0,0.0690559983253479
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,1024,1,0,0.08201919794082642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,2048,1,0,0.09458879828453064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,4096,1,0,0.11922880411148071
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,8192,1,0,0.1674399971961975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,32,16384,1,0,0.2626240015029907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,4,1,0,0.05847679972648621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,8,1,0,0.05876799821853638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,16,1,0,0.058950400352478026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,32,1,0,0.058575999736785886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,64,1,0,0.06018239855766296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,128,1,0,0.062243199348449706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,256,1,0,0.07286400198936463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,512,1,0,0.08462079763412475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,1024,1,0,0.10008319616317748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,2048,1,0,0.1254464030265808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,4096,1,0,0.1732800006866455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,8192,1,0,0.26889920234680176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,64,16384,1,0,0.4612576007843018
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,128,4,1,0,0.0630079984664917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,128,8,1,0,0.06304640173912049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,128,16,1,0,0.06348479986190796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,128,32,1,0,0.06400319933891296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,128,64,1,0,0.06593919992446899
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,128,128,1,0,0.06801599860191346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,128,256,1,0,0.09104959964752198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,128,512,1,0,0.11153279542922974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,128,1024,1,0,0.136188805103302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,128,2048,1,0,0.18596479892730713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,128,4096,1,0,0.28324160575866697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,128,8192,1,0,0.4768832206726074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,256,4,1,0,0.07649919986724854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,256,8,1,0,0.07654719948768615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,256,16,1,0,0.07723519802093506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,256,32,1,0,0.07848320007324219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,256,64,1,0,0.08194559812545776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,256,128,1,0,0.08851519823074341
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,256,256,1,0,0.1043936014175415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,256,512,1,0,0.1306399941444397
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,256,1024,1,0,0.1789407968521118
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,256,2048,1,0,0.2755520105361938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,256,4096,1,0,0.47116479873657224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,512,4,1,0,0.11041280031204223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,512,8,1,0,0.1111456036567688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,512,16,1,0,0.11385279893875122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,512,32,1,0,0.11776959896087646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,512,64,1,0,0.12482240200042724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,512,128,1,0,0.13760319948196412
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,512,256,1,0,0.16531200408935548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,512,512,1,0,0.21765120029449464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,512,1024,1,0,0.3148864030838013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,512,2048,1,0,0.507916784286499
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1024,4,1,0,0.19548159837722778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1024,8,1,0,0.19613759517669677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1024,16,1,0,0.19924160242080688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1024,32,1,0,0.2041696071624756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1024,64,1,0,0.21487679481506347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1024,128,1,0,0.23623039722442626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1024,256,1,0,0.28976318836212156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1024,512,1,0,0.39290881156921387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,32,1024,1024,1,0,0.588156795501709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,4,1,0,0.04102720022201538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,8,1,0,0.0411296010017395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,16,1,0,0.041417598724365234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,32,1,0,0.04370239973068237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,64,1,0,0.043059200048446655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,128,1,0,0.04287999868392944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,256,1,0,0.04402559995651245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,512,1,0,0.045788800716400145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,1024,1,0,0.050271999835968015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,2048,1,0,0.05050879716873169
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,4096,1,0,0.05230399966239929
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,8192,1,0,0.0596671998500824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1,16384,1,0,0.07295039892196656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,4,1,0,0.04100480079650879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,8,1,0,0.041017600893974306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,16,1,0,0.0412992000579834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,32,1,0,0.043558400869369504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,64,1,0,0.043094399571418765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,128,1,0,0.04287360012531281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,256,1,0,0.043926399946212766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,512,1,0,0.046172800660133365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,1024,1,0,0.04636160135269165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,2048,1,0,0.04867840111255646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,4096,1,0,0.05378239750862122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,8192,1,0,0.0642911970615387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,2,16384,1,0,0.07811840176582337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,4,1,0,0.04081920087337494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,8,1,0,0.040956801176071166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,16,1,0,0.04116159975528717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,32,1,0,0.04392000138759613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,64,1,0,0.04321280121803284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,128,1,0,0.042828801274299624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,256,1,0,0.04383679926395416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,512,1,0,0.04416320025920868
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,1024,1,0,0.047007998824119566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,2048,1,0,0.051622402667999265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,4096,1,0,0.05920000076293945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,8192,1,0,0.07084800004959106
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,4,16384,1,0,0.08304960131645203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,4,1,0,0.039843198657035825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,8,1,0,0.0397599995136261
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,16,1,0,0.04008319973945618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,32,1,0,0.0421887993812561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,64,1,0,0.041791999340057374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,128,1,0,0.04185279905796051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,256,1,0,0.041791999340057374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,512,1,0,0.043996798992156985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,1024,1,0,0.04832000136375427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,2048,1,0,0.05649920105934143
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,4096,1,0,0.06983680129051209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,8192,1,0,0.08232319951057435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,8,16384,1,0,0.10668480396270752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,4,1,0,0.03938559889793396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,8,1,0,0.039552000164985654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,16,1,0,0.039657598733901976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,32,1,0,0.04252800047397613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,64,1,0,0.04169279932975769
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,128,1,0,0.04195519983768463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,256,1,0,0.04365119934082031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,512,1,0,0.04809280037879944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,1024,1,0,0.055580800771713255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,2048,1,0,0.06655359864234925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,4096,1,0,0.07958719730377198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,8192,1,0,0.1041759967803955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,16,16384,1,0,0.15299199819564818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,4,1,0,0.03953920006752014
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,8,1,0,0.039673599600791934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,16,1,0,0.039740800857543945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,32,1,0,0.042607998847961424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,64,1,0,0.04305599927902222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,128,1,0,0.045465600490570066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,256,1,0,0.048982399702072146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,512,1,0,0.057017600536346434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,1024,1,0,0.06779839992523193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,2048,1,0,0.08216000199317933
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,4096,1,0,0.10652799606323242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,8192,1,0,0.15446720123291016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,32,16384,1,0,0.2503040075302124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,4,1,0,0.03770560026168823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,8,1,0,0.03788160085678101
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,16,1,0,0.03872320055961609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,32,1,0,0.03854719996452331
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,64,1,0,0.04110719859600067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,128,1,0,0.045535999536514285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,256,1,0,0.05668799877166748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,512,1,0,0.0709824025630951
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,1024,1,0,0.08616960048675537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,2048,1,0,0.11109119653701782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,4096,1,0,0.15958399772644044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,8192,1,0,0.25516159534454347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,64,16384,1,0,0.44608640670776367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,4,1,0,0.043017598986625674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,8,1,0,0.0434112012386322
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,16,1,0,0.04514879882335663
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,32,1,0,0.04531520009040833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,64,1,0,0.04937599897384644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,128,1,0,0.056796801090240476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,256,1,0,0.0745024025440216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,512,1,0,0.09416639804840088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,1024,1,0,0.11829119920730591
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,2048,1,0,0.16811200380325317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,4096,1,0,0.26541121006011964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,8192,1,0,0.45661439895629885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,128,16384,1,0,0.8618783950805664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,256,4,1,0,0.059926402568817136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,256,8,1,0,0.060438400506973265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,256,16,1,0,0.061484801769256595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,256,32,1,0,0.06319040060043335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,256,64,1,0,0.06749759912490845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,256,128,1,0,0.07379199862480164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,256,256,1,0,0.0882207989692688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,256,512,1,0,0.11340160369873047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,256,1024,1,0,0.16240960359573364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,256,2048,1,0,0.2599423885345459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,256,4096,1,0,0.45218238830566404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,256,8192,1,0,0.8383744239807129
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,512,4,1,0,0.08536000251770019
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,512,8,1,0,0.0864736020565033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,512,16,1,0,0.08749120235443116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,512,32,1,0,0.09093120098114013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,512,64,1,0,0.09861119985580444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,512,128,1,0,0.11180800199508667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,512,256,1,0,0.13874239921569825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,512,512,1,0,0.19000639915466308
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,512,1024,1,0,0.28654720783233645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,512,2048,1,0,0.4804831981658936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,512,4096,1,0,0.866051197052002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1024,4,1,0,0.1326624035835266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1024,8,1,0,0.13514560461044312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1024,16,1,0,0.14011520147323608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1024,32,1,0,0.14696639776229858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1024,64,1,0,0.15597440004348756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1024,128,1,0,0.1785920023918152
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1024,256,1,0,0.2306879997253418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1024,512,1,0,0.33223040103912355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1024,1024,1,0,0.5262944221496582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,flashinfer,fp8_block,fp8,16,1024,2048,1,0,0.9143520355224609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,4,1,0,0.1105728030204773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,8,1,0,0.11106239557266236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,16,1,0,0.11055359840393067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,32,1,0,0.11102080345153809
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,64,1,0,0.11229439973831176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,128,1,0,0.11410239934921265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,256,1,0,0.11818879842758179
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,512,1,0,0.12070720195770264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,1024,1,0,0.12137919664382935
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,2048,1,0,0.12249599695205689
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,4096,1,0,0.1264863967895508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,8192,1,0,0.13044159412384032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,1,16384,1,0,0.13600640296936034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,4,1,0,0.11271040439605713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,8,1,0,0.11280640363693237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,16,1,0,0.11256639957427979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,32,1,0,0.11293120384216308
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,64,1,0,0.11434240341186523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,128,1,0,0.11570240259170532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,256,1,0,0.11950399875640869
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,512,1,0,0.12155519723892212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,1024,1,0,0.12232639789581298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,2048,1,0,0.1264415979385376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,4096,1,0,0.13048319816589354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,8192,1,0,0.13571840524673462
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,2,16384,1,0,0.1443711996078491
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,4,1,0,0.11324160099029541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,8,1,0,0.11370879411697388
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,16,1,0,0.11360000371932984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,32,1,0,0.11360319852828979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,64,1,0,0.11536320447921752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,128,1,0,0.11664639711380005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,256,1,0,0.12305279970169067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,512,1,0,0.12372479438781739
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,1024,1,0,0.12836159467697145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,2048,1,0,0.13182400465011596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,4096,1,0,0.13674559593200683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,8192,1,0,0.14655040502548217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,4,16384,1,0,0.16219199895858766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,4,1,0,0.11440320014953613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,8,1,0,0.11422079801559448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,16,1,0,0.11489599943161011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,32,1,0,0.11447999477386475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,64,1,0,0.1147104024887085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,128,1,0,0.1163167953491211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,256,1,0,0.1265056014060974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,512,1,0,0.1283136010169983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,1024,1,0,0.13128639459609986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,2048,1,0,0.13683520555496215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,4096,1,0,0.14716479778289795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,8192,1,0,0.1609279990196228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,8,16384,1,0,0.18860479593276977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,16,4,1,0,0.11546560525894164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,16,8,1,0,0.11543680429458618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,16,16,1,0,0.11512960195541382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,16,32,1,0,0.11594560146331787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,16,64,1,0,0.11728320121765137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,16,128,1,0,0.11902719736099243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,16,256,1,0,0.1321727991104126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,16,512,1,0,0.13451839685440065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,16,1024,1,0,0.14008959531784057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,16,2048,1,0,0.14911680221557616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,32,4,1,0,0.11590399742126464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,32,8,1,0,0.11592960357666016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,32,16,1,0,0.11582720279693604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,32,32,1,0,0.11590720415115356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,32,64,1,0,0.11763199567794799
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,32,128,1,0,0.1189568042755127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,32,256,1,0,0.13852800130844117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,32,512,1,0,0.14182720184326172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,32,1024,1,0,0.15079679489135742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,64,4,1,0,0.12253439426422119
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,64,8,1,0,0.12291519641876221
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,64,16,1,0,0.12187199592590332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,64,32,1,0,0.12341760396957398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,64,64,1,0,0.124508798122406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,64,128,1,0,0.126528000831604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,64,256,1,0,0.13581759929656984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,64,512,1,0,0.1436576008796692
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,128,4,1,0,0.14091839790344238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,128,8,1,0,0.1415552020072937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,128,16,1,0,0.14089280366897583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,128,32,1,0,0.14175679683685302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,128,64,1,0,0.14528000354766846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,128,128,1,0,0.15035840272903442
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,128,256,1,0,0.16998720169067383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,256,4,1,0,0.193014395236969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,256,8,1,0,0.19377280473709108
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,256,16,1,0,0.19490879774093628
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,256,32,1,0,0.19600000381469726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,256,64,1,0,0.20071039199829102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,256,128,1,0,0.2086944103240967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,512,4,1,0,0.3322655916213989
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,512,8,1,0,0.3344352006912231
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,512,16,1,0,0.33641600608825684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,512,32,1,0,0.33804481029510497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,512,64,1,0,0.34444479942321776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,128,512,128,1,0,0.3598848104476929
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,4,1,0,0.07730559706687927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,8,1,0,0.07716159820556641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,16,1,0,0.07739840149879455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,32,1,0,0.0775327980518341
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,64,1,0,0.0790336012840271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,128,1,0,0.08057600259780884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,256,1,0,0.0846015989780426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,512,1,0,0.08600320219993592
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,1024,1,0,0.0867904007434845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,2048,1,0,0.08876799941062927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,4096,1,0,0.09031040072441102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,8192,1,0,0.09614400267601013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1,16384,1,0,0.1007807970046997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,4,1,0,0.07650240063667298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,8,1,0,0.07659839987754821
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,16,1,0,0.07626559734344482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,32,1,0,0.07676799893379212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,64,1,0,0.07782400250434876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,128,1,0,0.07987840175628662
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,256,1,0,0.08275520205497741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,512,1,0,0.08481280207633972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,1024,1,0,0.08626880049705506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,2048,1,0,0.08799999952316284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,4096,1,0,0.09305599927902222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,8192,1,0,0.0992031991481781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,2,16384,1,0,0.10620800256729127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,4,1,0,0.0778335988521576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,8,1,0,0.07720959782600403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,16,1,0,0.07794880270957946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,32,1,0,0.07781760096549988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,64,1,0,0.07950080037117005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,128,1,0,0.08065919876098633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,256,1,0,0.08569599986076355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,512,1,0,0.08678399920463561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,1024,1,0,0.08901439905166626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,2048,1,0,0.09395840167999267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,4096,1,0,0.09879040122032165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,8192,1,0,0.10558079481124878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,4,16384,1,0,0.1195904016494751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,4,1,0,0.0765504002571106
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,8,1,0,0.07740479707717896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,16,1,0,0.07668799757957459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,32,1,0,0.07696319818496704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,64,1,0,0.07786880135536194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,128,1,0,0.07994880080223084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,256,1,0,0.08721280097961426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,512,1,0,0.0883679986000061
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,1024,1,0,0.09417920112609864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,2048,1,0,0.09835839867591858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,4096,1,0,0.10461440086364746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,8192,1,0,0.11912000179290771
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,8,16384,1,0,0.1468224048614502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,4,1,0,0.0762719988822937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,8,1,0,0.07690879702568054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,16,1,0,0.07671999931335449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,32,1,0,0.07679359912872315
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,64,1,0,0.07879040241241456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,128,1,0,0.08007360100746155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,256,1,0,0.09122560024261475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,512,1,0,0.09391040205955506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,1024,1,0,0.09770879745483399
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,2048,1,0,0.10581439733505249
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,4096,1,0,0.12018879652023315
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,8192,1,0,0.14388480186462402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,16,16384,1,0,0.194649600982666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,4,1,0,0.0800320029258728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,8,1,0,0.07956159710884095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,16,1,0,0.07927359938621521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,32,1,0,0.07997120022773743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,64,1,0,0.08142079710960388
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,128,1,0,0.08315839767456054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,256,1,0,0.09749760031700135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,512,1,0,0.10080640316009522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,1024,1,0,0.10967040061950684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,2048,1,0,0.12258239984512329
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,4096,1,0,0.1469823956489563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,8192,1,0,0.19806079864501952
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,32,16384,1,0,0.29980480670928955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,4,1,0,0.08371520042419434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,8,1,0,0.08353279829025269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,16,1,0,0.08383679986000062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,32,1,0,0.08400319814682007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,64,1,0,0.0851360023021698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,128,1,0,0.08746880292892456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,256,1,0,0.10727039575576783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,512,1,0,0.11498880386352539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,1024,1,0,0.12879680395126342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,2048,1,0,0.15291199684143067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,4096,1,0,0.2061568021774292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,8192,1,0,0.3071455955505371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,64,16384,1,0,0.5058112144470215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,128,4,1,0,0.09303039908409119
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,128,8,1,0,0.09211199879646301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,128,16,1,0,0.09207680225372314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,128,32,1,0,0.09263359904289245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,128,64,1,0,0.09473279714584351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,128,128,1,0,0.09724799990653991
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,128,256,1,0,0.11133760213851929
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,128,512,1,0,0.12541439533233642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,128,1024,1,0,0.150655996799469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,128,2048,1,0,0.20265278816223145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,128,4096,1,0,0.3031519889831543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,128,8192,1,0,0.503382396697998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,256,4,1,0,0.1130944013595581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,256,8,1,0,0.1136512041091919
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,256,16,1,0,0.1144927978515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,256,32,1,0,0.11545920372009277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,256,64,1,0,0.12055039405822754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,256,128,1,0,0.1281888008117676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,256,256,1,0,0.15501760244369506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,256,512,1,0,0.18226560354232788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,256,1024,1,0,0.2325439929962158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,256,2048,1,0,0.33195838928222654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,256,4096,1,0,0.5287775993347168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,512,4,1,0,0.1857632040977478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,512,8,1,0,0.185862398147583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,512,16,1,0,0.18730239868164061
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,512,32,1,0,0.19011520147323607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,512,64,1,0,0.19640640020370484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,512,128,1,0,0.2098207950592041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,512,256,1,0,0.2593503952026367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,512,512,1,0,0.3130111932754517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,512,1024,1,0,0.4107200145721436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,512,2048,1,0,0.6040095806121826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1024,4,1,0,0.3292959928512573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1024,8,1,0,0.33070080280303954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1024,16,1,0,0.3320672035217285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1024,32,1,0,0.33035199642181395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1024,64,1,0,0.3444256067276001
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1024,128,1,0,0.36795840263366697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1024,256,1,0,0.46072959899902344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1024,512,1,0,0.5634528160095215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,64,1024,1024,1,0,0.755129623413086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,4,1,0,0.06024320125579834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,8,1,0,0.06023359894752502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,16,1,0,0.06000959873199463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,32,1,0,0.06023359894752502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,64,1,0,0.06188160181045532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,128,1,0,0.06385599970817565
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,256,1,0,0.06624320149421692
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,512,1,0,0.06818240284919738
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,1024,1,0,0.06845759749412536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,2048,1,0,0.0698303997516632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,4096,1,0,0.0723904013633728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,8192,1,0,0.07582399845123292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1,16384,1,0,0.08139200210571289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,4,1,0,0.060099202394485476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,8,1,0,0.060192000865936277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,16,1,0,0.06023359894752502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,32,1,0,0.06047359704971313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,64,1,0,0.061868798732757566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,128,1,0,0.0630944013595581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,256,1,0,0.06537600159645081
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,512,1,0,0.06798080205917359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,1024,1,0,0.06844800114631652
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,2048,1,0,0.07005119919776917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,4096,1,0,0.07327359914779663
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,8192,1,0,0.07776640057563781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,2,16384,1,0,0.0848800003528595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,4,1,0,0.06034240126609802
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,8,1,0,0.06030719876289368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,16,1,0,0.059913599491119386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,32,1,0,0.06033920049667359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,64,1,0,0.061638402938842776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,128,1,0,0.06345599889755249
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,256,1,0,0.06625599861145019
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,512,1,0,0.0690496027469635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,1024,1,0,0.07043520212173462
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,2048,1,0,0.07326400279998779
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,4096,1,0,0.07698559761047363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,8192,1,0,0.08354880213737488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,4,16384,1,0,0.09786559939384461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,4,1,0,0.05820479989051819
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,8,1,0,0.05790719985961914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,16,1,0,0.05836799740791321
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,32,1,0,0.05840319991111755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,64,1,0,0.059996801614761355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,128,1,0,0.06149439811706543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,256,1,0,0.06619840264320373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,512,1,0,0.06774399876594543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,1024,1,0,0.07060800194740295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,2048,1,0,0.0745311975479126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,4096,1,0,0.08123520016670227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,8192,1,0,0.0954688012599945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,8,16384,1,0,0.12216639518737793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,4,1,0,0.05899199843406677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,8,1,0,0.05887680053710938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,16,1,0,0.058963197469711306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,32,1,0,0.05904960036277771
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,64,1,0,0.06042240262031555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,128,1,0,0.062134397029876706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,256,1,0,0.06892160177230836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,512,1,0,0.07100160121917724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,1024,1,0,0.07482240200042725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,2048,1,0,0.0822272002696991
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,4096,1,0,0.09615359902381897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,8192,1,0,0.1216480016708374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,16,16384,1,0,0.1742527961730957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,4,1,0,0.06064959764480591
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,8,1,0,0.060464000701904295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,16,1,0,0.060601598024368285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,32,1,0,0.06104320287704468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,64,1,0,0.0628063976764679
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,128,1,0,0.06456000208854676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,256,1,0,0.07395200133323669
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,512,1,0,0.07724800109863281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,1024,1,0,0.08385599851608276
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,2048,1,0,0.09841279983520508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,4096,1,0,0.12299840450286866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,8192,1,0,0.17462079524993895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,32,16384,1,0,0.2750880002975464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,4,1,0,0.06267840266227723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,8,1,0,0.06218879818916321
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,16,1,0,0.06299200057983398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,32,1,0,0.06347839832305908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,64,1,0,0.06537600159645081
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,128,1,0,0.06684160232543945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,256,1,0,0.07996479868888855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,512,1,0,0.08660159707069397
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,1024,1,0,0.10154880285263061
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,2048,1,0,0.12672959566116332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,4096,1,0,0.17817280292510987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,8192,1,0,0.27768959999084475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,64,16384,1,0,0.47817602157592776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,128,4,1,0,0.06800959706306457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,128,8,1,0,0.06820480227470398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,128,16,1,0,0.06840959787368775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,128,32,1,0,0.06920639872550964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,128,64,1,0,0.07221119999885559
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,128,128,1,0,0.07472320199012757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,128,256,1,0,0.0872160017490387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,128,512,1,0,0.10154240131378174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,128,1024,1,0,0.1260032057762146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,128,2048,1,0,0.17800960540771485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,128,4096,1,0,0.2783936023712158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,128,8192,1,0,0.4791103839874268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,256,4,1,0,0.08340479731559754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,256,8,1,0,0.08327999711036682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,256,16,1,0,0.083433598279953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,256,32,1,0,0.08489599823951721
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,256,64,1,0,0.08928959965705871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,256,128,1,0,0.0969760000705719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,256,256,1,0,0.12072960138320923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,256,512,1,0,0.14795199632644654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,256,1024,1,0,0.1977568030357361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,256,2048,1,0,0.2962143898010254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,256,4096,1,0,0.493123197555542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,512,4,1,0,0.12075519561767578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,512,8,1,0,0.12118079662322997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,512,16,1,0,0.12257920503616333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,512,32,1,0,0.12631360292434693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,512,64,1,0,0.13556480407714844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,512,128,1,0,0.14941120147705078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,512,256,1,0,0.19160319566726686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,512,512,1,0,0.2439039945602417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,512,1024,1,0,0.34138240814208987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,512,2048,1,0,0.5351168155670166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1024,4,1,0,0.20615360736846924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1024,8,1,0,0.20819199085235596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1024,16,1,0,0.21031360626220702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1024,32,1,0,0.21311359405517577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1024,64,1,0,0.22491519451141356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1024,128,1,0,0.2487488031387329
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1024,256,1,0,0.3235872030258179
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1024,512,1,0,0.42812161445617675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,32,1024,1024,1,0,0.6217088222503662
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,4,1,0,0.044787201285362246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,8,1,0,0.04479359984397888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,16,1,0,0.0447488009929657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,32,1,0,0.04483200013637543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,64,1,0,0.04628480076789856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,128,1,0,0.04796159863471985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,256,1,0,0.04837439954280853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,512,1,0,0.049798399209976196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,1024,1,0,0.05102400183677673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,2048,1,0,0.05348160266876221
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,4096,1,0,0.05594559907913208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,8192,1,0,0.058620798587799075
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1,16384,1,0,0.06818559765815735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,4,1,0,0.04560959935188293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,8,1,0,0.045865601301193236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,16,1,0,0.045561599731445315
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,32,1,0,0.04556800127029419
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,64,1,0,0.04736959934234619
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,128,1,0,0.048614400625228885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,256,1,0,0.04896959960460663
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,512,1,0,0.05135040283203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,1024,1,0,0.05257920026779175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,2048,1,0,0.05380480289459229
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,4096,1,0,0.05824319720268249
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,8192,1,0,0.0668287992477417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,2,16384,1,0,0.07582719922065735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,4,1,0,0.04472959935665131
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,8,1,0,0.04490239918231964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,16,1,0,0.044870400428771974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,32,1,0,0.04464319944381714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,64,1,0,0.04656639993190766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,128,1,0,0.047932800650596616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,256,1,0,0.04981119930744171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,512,1,0,0.05159040093421936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,1024,1,0,0.052095997333526614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,2048,1,0,0.05655999779701233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,4096,1,0,0.06494399905204773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,8192,1,0,0.07307839989662171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,4,16384,1,0,0.0871616005897522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,4,1,0,0.04308800101280212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,8,1,0,0.043136000633239746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,16,1,0,0.04325119853019714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,32,1,0,0.04337919950485229
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,64,1,0,0.04473919868469238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,128,1,0,0.04657280147075653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,256,1,0,0.048179200291633605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,512,1,0,0.05000320076942444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,1024,1,0,0.0547104001045227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,2048,1,0,0.06312000155448913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,4096,1,0,0.07005119919776917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,8192,1,0,0.08461440205574036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,8,16384,1,0,0.11089919805526734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,4,1,0,0.04259519875049591
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,8,1,0,0.042982399463653564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,16,1,0,0.04301440119743347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,32,1,0,0.04343999922275543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,64,1,0,0.04472320079803467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,128,1,0,0.04644800126552582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,256,1,0,0.05170239806175232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,512,1,0,0.05623360276222229
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,1024,1,0,0.06323199868202209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,2048,1,0,0.0710207998752594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,4096,1,0,0.08496639728546143
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,8192,1,0,0.11080960035324097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,16,16384,1,0,0.16102720499038697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,4,1,0,0.043696001172065735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,8,1,0,0.0441760003566742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,16,1,0,0.04441919922828674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,32,1,0,0.04578559994697571
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,64,1,0,0.048102399706840514
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,128,1,0,0.050307202339172366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,256,1,0,0.057180798053741454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,512,1,0,0.0648256003856659
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,1024,1,0,0.0722656011581421
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,2048,1,0,0.0855840027332306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,4096,1,0,0.11135679483413696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,8192,1,0,0.1619488000869751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,32,16384,1,0,0.26376640796661377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,64,4,1,0,0.04410240054130554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,64,8,1,0,0.046265599131584165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,64,16,1,0,0.04723199903964996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,64,32,1,0,0.04621120095252991
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,64,64,1,0,0.04867199957370758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,64,128,1,0,0.05342400074005127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,64,256,1,0,0.06715199947357178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,64,512,1,0,0.07443199753761291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,64,1024,1,0,0.08718079924583436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,64,2048,1,0,0.1134112000465393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,64,4096,1,0,0.16433919668197633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,64,8192,1,0,0.2645024061203003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,128,4,1,0,0.052527999877929686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,128,8,1,0,0.0511680006980896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,128,16,1,0,0.051551997661590576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,128,32,1,0,0.05303679704666138
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,128,64,1,0,0.05748479962348938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,128,128,1,0,0.06413120031356812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,128,256,1,0,0.07612159848213196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,128,512,1,0,0.08970239758491516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,128,1024,1,0,0.11507840156555176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,128,2048,1,0,0.1665120005607605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,128,4096,1,0,0.26600639820098876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,256,4,1,0,0.06726719737052918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,256,8,1,0,0.06687679886817932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,256,16,1,0,0.06804479956626892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,256,32,1,0,0.0695904016494751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,256,64,1,0,0.0748799979686737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,256,128,1,0,0.08194559812545776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,256,256,1,0,0.10419520139694213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,256,512,1,0,0.13091200590133667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,256,1024,1,0,0.1807968020439148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,256,2048,1,0,0.2785056114196777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,512,4,1,0,0.09811519980430602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,512,8,1,0,0.09790719747543335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,512,16,1,0,0.09899839758872986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,512,32,1,0,0.10080959796905517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,512,64,1,0,0.11123199462890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,512,128,1,0,0.12489279508590698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,512,256,1,0,0.16312639713287352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,512,512,1,0,0.21590399742126465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,512,1024,1,0,0.3125823974609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1024,4,1,0,0.14740480184555055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1024,8,1,0,0.14850879907608033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1024,16,1,0,0.1509152054786682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1024,32,1,0,0.15456960201263428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1024,64,1,0,0.17119359970092773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1024,128,1,0,0.19431999921798707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1024,256,1,0,0.25998721122741697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,mla_generation,fa3,fp8_block,fp8,16,1024,512,1,0,0.36448960304260253
