framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,1,128,1,float16,float16,0,91.34024658203126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,1,128,1,float16,fp8,0,78.97162475585938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,1,128,1,fp8,fp8,0,77.1694091796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,2,128,1,float16,fp8,0,76.76019287109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,2,128,1,fp8,fp8,0,77.45869750976563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,4,128,1,float16,fp8,0,77.95930786132813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,2,128,1,float16,float16,0,91.99368286132812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,4,128,1,float16,float16,0,93.05631713867187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,96,128,1,float16,fp8,0,45.077450561523435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,4,128,1,fp8,fp8,0,78.04912109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,96,128,1,float16,float16,0,52.55149536132812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,96,128,1,fp8,fp8,0,44.84516906738281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,1,128,1,float16,float16,0,45.9621337890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,8,128,1,float16,fp8,0,78.35524291992188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,8,128,1,float16,float16,0,94.9795166015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,8,128,1,fp8,fp8,0,77.59135131835937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,1,128,1,float16,fp8,0,38.879452514648435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,1,128,1,fp8,fp8,0,38.71460876464844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,2,128,1,float16,fp8,0,38.70197448730469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,2,128,1,float16,float16,0,45.63856506347656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,2,128,1,fp8,fp8,0,38.826971435546874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,4,128,1,fp8,fp8,0,38.11259155273437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,4,128,1,float16,fp8,0,39.95238342285156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,4,128,1,float16,float16,0,46.41268310546875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,96,128,1,float16,fp8,0,22.56092529296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,96,128,1,float16,float16,0,26.95319519042969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,8,128,1,float16,fp8,0,39.3639404296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,8,128,1,float16,float16,0,46.69902648925781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,8,128,1,fp8,fp8,0,39.287985229492186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,96,128,1,fp8,fp8,0,21.879808044433595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,1,128,1,float16,float16,0,23.452386474609376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,1,128,1,float16,fp8,0,18.75887451171875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,1,128,1,fp8,fp8,0,19.14824676513672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,2,128,1,float16,fp8,0,19.163847351074217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,2,128,1,float16,float16,0,22.559642028808593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,2,128,1,fp8,fp8,0,18.977760314941406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,4,128,1,float16,fp8,0,19.800152587890626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,4,128,1,fp8,fp8,0,19.54727783203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,4,128,1,float16,float16,0,22.968215942382812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,96,128,1,float16,float16,0,13.27021484375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,8,128,1,float16,fp8,0,19.724501037597655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,96,128,1,float16,fp8,0,11.563003540039062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,8,128,1,float16,float16,0,23.591337585449217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,8,128,1,fp8,fp8,0,19.833895874023437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,96,128,1,fp8,fp8,0,10.948248291015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,1,128,1,float16,fp8,0,9.528959655761719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,1,128,1,float16,float16,0,12.043569946289063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,1,128,1,fp8,fp8,0,9.310794830322266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,2,128,1,float16,fp8,0,9.835603332519531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,2,128,1,float16,float16,0,11.483039855957031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,2,128,1,fp8,fp8,0,9.273097229003906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,4,128,1,float16,float16,0,11.67493133544922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,4,128,1,float16,fp8,0,9.400787353515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,4,128,1,fp8,fp8,0,9.479347229003906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,8,128,1,float16,fp8,0,9.61195068359375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,8,128,1,fp8,fp8,0,9.535619354248047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,8,128,1,float16,float16,0,11.904860687255859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,1,128,1,float16,fp8,0,44.070761108398436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,1,128,1,fp8,fp8,0,44.021435546875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,2,128,1,float16,fp8,0,44.16356506347656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,1,128,1,float16,float16,0,51.862255859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,2,128,1,fp8,fp8,0,44.03067321777344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,4,128,1,float16,fp8,0,45.561868286132814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,2,128,1,float16,float16,0,51.7332275390625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,4,128,1,float16,float16,0,52.650811767578126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,96,128,1,float16,fp8,0,27.316134643554687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,96,128,1,float16,float16,0,31.110311889648436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,96,128,1,fp8,fp8,0,26.162945556640626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,1,128,1,float16,float16,0,25.83575439453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,4,128,1,fp8,fp8,0,44.399786376953124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,8,128,1,float16,fp8,0,44.78968505859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,8,128,1,fp8,fp8,0,44.937066650390626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,8,128,1,float16,float16,0,53.17462158203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,1,128,1,float16,fp8,0,22.187864685058592
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,1,128,1,fp8,fp8,0,21.962779235839843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,2,128,1,float16,float16,0,26.189791870117187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,2,128,1,float16,fp8,0,22.376504516601564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,2,128,1,fp8,fp8,0,22.271763610839844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,4,128,1,float16,fp8,0,22.10694122314453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,4,128,1,float16,float16,0,26.204983520507813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,4,128,1,fp8,fp8,0,21.817660522460937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,96,128,1,float16,fp8,0,13.184371948242188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,96,128,1,float16,float16,0,15.877371215820313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,8,128,1,float16,fp8,0,22.405641174316408
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,8,128,1,fp8,fp8,0,22.25652160644531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,8,128,1,float16,float16,0,27.4306884765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,96,128,1,fp8,fp8,0,13.405299377441406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,1,128,1,float16,fp8,0,10.815325164794922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,1,128,1,float16,float16,0,12.887315368652343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,1,128,1,fp8,fp8,0,11.001771545410156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,2,128,1,float16,fp8,0,11.07996826171875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,2,128,1,float16,float16,0,13.009596252441407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,2,128,1,fp8,fp8,0,10.91779022216797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,4,128,1,float16,fp8,0,10.959921264648438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,4,128,1,fp8,fp8,0,11.167835235595703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,4,128,1,float16,float16,0,13.299772644042969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,8,128,1,float16,float16,0,13.215992736816407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,96,128,1,float16,float16,0,7.894798278808594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,8,128,1,float16,fp8,0,11.318555450439453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,96,128,1,float16,fp8,0,6.569814300537109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,8,128,1,fp8,fp8,0,11.083838653564452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,96,128,1,fp8,fp8,0,6.596119689941406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,1,128,1,float16,float16,0,6.455830383300781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,1,128,1,float16,fp8,0,5.5139312744140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,1,128,1,fp8,fp8,0,5.368431854248047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,2,128,1,float16,fp8,0,5.411500930786133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,2,128,1,float16,float16,0,6.5381614685058596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,2,128,1,fp8,fp8,0,5.465752029418946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,4,128,1,float16,fp8,0,5.55645751953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,4,128,1,fp8,fp8,0,5.669142532348633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,4,128,1,float16,float16,0,6.6319091796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,8,128,1,float16,fp8,0,5.498990249633789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,8,128,1,float16,float16,0,6.84443359375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,8,128,1,fp8,fp8,0,5.609254455566406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,1,128,1,float16,fp8,0,30.805224609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,1,128,1,fp8,fp8,0,31.629736328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,2,128,1,float16,fp8,0,31.193670654296874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,2,128,1,fp8,fp8,0,30.955935668945312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,4,128,1,float16,fp8,0,29.93230285644531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,1,128,1,float16,float16,0,36.58426513671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,2,128,1,float16,float16,0,36.51134033203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,4,128,1,float16,float16,0,37.196710205078126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,96,128,1,float16,fp8,0,19.25206756591797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,96,128,1,fp8,fp8,0,19.427735900878908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,96,128,1,float16,float16,0,23.113844299316405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,1,128,1,float16,float16,0,18.339398193359376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,4,128,1,fp8,fp8,0,30.8158935546875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,8,128,1,float16,fp8,0,31.577761840820312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,8,128,1,fp8,fp8,0,31.332879638671876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,8,128,1,float16,float16,0,37.71296691894531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,1,128,1,float16,fp8,0,15.503761291503906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,1,128,1,fp8,fp8,0,15.297454833984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,2,128,1,float16,fp8,0,15.501023864746093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,2,128,1,float16,float16,0,17.679286193847656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,2,128,1,fp8,fp8,0,15.708888244628906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,4,128,1,float16,fp8,0,15.607241821289062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,4,128,1,float16,float16,0,18.791734313964845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,4,128,1,fp8,fp8,0,15.941082763671876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,96,128,1,float16,fp8,0,9.510387420654297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,96,128,1,float16,float16,0,11.866942596435546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,8,128,1,float16,fp8,0,15.743637084960938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,8,128,1,float16,float16,0,18.5694580078125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,8,128,1,fp8,fp8,0,15.898989868164062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,96,128,1,fp8,fp8,0,9.59879379272461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,1,128,1,float16,float16,0,9.0731201171875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,1,128,1,float16,fp8,0,7.794985961914063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,1,128,1,fp8,fp8,0,7.642534637451172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,2,128,1,float16,fp8,0,7.7698112487792965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,2,128,1,float16,float16,0,8.91744613647461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,2,128,1,fp8,fp8,0,7.609259033203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,4,128,1,float16,fp8,0,7.700736236572266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,4,128,1,float16,float16,0,9.341804504394531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,4,128,1,fp8,fp8,0,7.6135986328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,96,128,1,float16,fp8,0,4.801612854003906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,96,128,1,float16,float16,0,5.402660751342774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,8,128,1,float16,fp8,0,7.880166625976562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,8,128,1,float16,float16,0,9.847988891601563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,96,128,1,fp8,fp8,0,4.845710372924804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,1,128,1,float16,float16,0,4.505847930908203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,8,128,1,fp8,fp8,0,7.801789093017578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,1,128,1,float16,fp8,0,3.8106369018554687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,1,128,1,fp8,fp8,0,3.7779441833496095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,2,128,1,float16,float16,0,4.272126388549805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,2,128,1,float16,fp8,0,3.8780479431152344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,2,128,1,fp8,fp8,0,3.9525310516357424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,4,128,1,fp8,fp8,0,3.8631935119628906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,4,128,1,float16,fp8,0,3.8567489624023437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,4,128,1,float16,float16,0,4.492086410522461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,8,128,1,float16,float16,0,4.965582275390625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,8,128,1,float16,fp8,0,3.9595806121826174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,8,128,1,fp8,fp8,0,3.961761474609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,1,128,1,fp8,fp8,0,40.296261596679685
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,2,128,1,float16,fp8,0,40.16529235839844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,1,128,1,float16,fp8,0,41.244198608398435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,2,128,1,fp8,fp8,0,40.60045166015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,4,128,1,float16,fp8,0,39.990780639648435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,1,128,1,float16,float16,0,47.33189392089844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,4,128,1,float16,float16,0,48.03424987792969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,2,128,1,float16,float16,0,48.158807373046876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,96,128,1,float16,fp8,0,26.056759643554688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,96,128,1,float16,float16,0,30.127035522460936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,1,128,1,float16,float16,0,23.154534912109376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,96,128,1,fp8,fp8,0,26.135055541992188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,4,128,1,fp8,fp8,0,40.776205444335936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,8,128,1,float16,fp8,0,42.285214233398435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,8,128,1,fp8,fp8,0,41.427783203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,8,128,1,float16,float16,0,48.34485778808594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,1,128,1,float16,fp8,0,20.184080505371092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,1,128,1,fp8,fp8,0,19.847593688964842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,2,128,1,float16,fp8,0,20.588673400878907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,2,128,1,float16,float16,0,24.609571838378905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,2,128,1,fp8,fp8,0,20.12200012207031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,4,128,1,float16,fp8,0,20.21009979248047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,4,128,1,float16,float16,0,24.690950012207033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,4,128,1,fp8,fp8,0,19.983070373535156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,8,128,1,float16,fp8,0,20.302082824707032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,96,128,1,float16,float16,0,15.463616943359375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,96,128,1,float16,fp8,0,13.188728332519531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,96,128,1,fp8,fp8,0,13.255929565429687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,8,128,1,float16,float16,0,24.442724609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,8,128,1,fp8,fp8,0,20.590997314453126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,1,128,1,float16,fp8,0,10.023506927490235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,1,128,1,float16,float16,0,12.089694213867187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,1,128,1,fp8,fp8,0,9.830001831054688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,2,128,1,float16,fp8,0,10.050939178466797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,2,128,1,float16,float16,0,12.24732666015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,2,128,1,fp8,fp8,0,9.912741088867188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,4,128,1,float16,fp8,0,10.330184173583984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,4,128,1,fp8,fp8,0,10.014756774902343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,4,128,1,float16,float16,0,12.38497314453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,96,128,1,float16,fp8,0,6.510736083984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,96,128,1,float16,float16,0,7.409003448486328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,8,128,1,float16,fp8,0,10.326576232910156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,1,128,1,float16,fp8,0,4.857489776611328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,1,128,1,float16,float16,0,6.1287487030029295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,8,128,1,float16,float16,0,12.082746887207032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,96,128,1,fp8,fp8,0,6.513072204589844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,8,128,1,fp8,fp8,0,10.322468566894532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,1,128,1,fp8,fp8,0,5.046495819091797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,2,128,1,float16,float16,0,5.5839073181152346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,2,128,1,float16,fp8,0,5.02807846069336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,2,128,1,fp8,fp8,0,4.878507232666015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,4,128,1,float16,fp8,0,4.914180755615234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,4,128,1,fp8,fp8,0,4.956563186645508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,4,128,1,float16,float16,0,6.21307373046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,8,128,1,float16,float16,0,6.488597106933594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,8,128,1,float16,fp8,0,5.108742523193359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,96,128,1,float16,fp8,0,3.2608448028564454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,1,128,1,float16,float16,0,2.8165584564208985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,96,128,1,float16,float16,0,3.7628543853759764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,96,128,1,fp8,fp8,0,3.250703811645508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,1,128,1,float16,fp8,0,2.54718074798584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,8,128,1,fp8,fp8,0,5.000372695922851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,1,128,1,fp8,fp8,0,2.4928144454956054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,2,128,1,float16,fp8,0,2.443252754211426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,2,128,1,float16,float16,0,2.8422895431518556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,2,128,1,fp8,fp8,0,2.7862064361572267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,4,128,1,float16,fp8,0,2.4758159637451174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,4,128,1,float16,float16,0,3.0593584060668944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,4,128,1,fp8,fp8,0,2.490190315246582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,8,128,1,float16,float16,0,2.802225685119629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,8,128,1,float16,fp8,0,2.7460575103759766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,8,128,1,fp8,fp8,0,2.5196367263793946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,1,128,1,fp8,fp8,0,22.91673889160156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,1,128,1,float16,fp8,0,23.149517822265626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,2,128,1,float16,fp8,0,23.621200561523438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,2,128,1,fp8,fp8,0,23.877291870117187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,4,128,1,float16,fp8,0,24.093049621582033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,1,128,1,float16,float16,0,27.28571472167969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,2,128,1,float16,float16,0,27.51395263671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,4,128,1,float16,float16,0,28.229400634765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,96,128,1,float16,fp8,0,16.173890686035158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,1,128,1,float16,float16,0,13.834536743164062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,96,128,1,fp8,fp8,0,16.26080322265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,96,128,1,float16,float16,0,18.875799560546874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,4,128,1,fp8,fp8,0,23.12255096435547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,8,128,1,fp8,fp8,0,23.533906555175783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,8,128,1,float16,fp8,0,24.56766357421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,8,128,1,float16,float16,0,28.128701782226564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,1,128,1,float16,fp8,0,11.382804870605469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,1,128,1,fp8,fp8,0,12.427220916748047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,2,128,1,float16,fp8,0,11.789536285400391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,2,128,1,fp8,fp8,0,11.570697784423828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,2,128,1,float16,float16,0,13.77660675048828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,4,128,1,float16,fp8,0,11.921369934082032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,4,128,1,float16,float16,0,14.439529418945312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,4,128,1,fp8,fp8,0,11.499890899658203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,96,128,1,float16,fp8,0,8.146587371826172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,96,128,1,float16,float16,0,9.616004943847656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,1,128,1,float16,fp8,0,5.7731281280517575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,8,128,1,float16,fp8,0,12.323623657226562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,8,128,1,fp8,fp8,0,11.866462707519531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,1,128,1,float16,float16,0,6.679785919189453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,96,128,1,fp8,fp8,0,8.312068939208984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,8,128,1,float16,float16,0,14.787156677246093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,1,128,1,fp8,fp8,0,5.789129638671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,2,128,1,float16,float16,0,6.732556915283203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,2,128,1,float16,fp8,0,5.850193786621094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,2,128,1,fp8,fp8,0,6.090836715698242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,4,128,1,float16,fp8,0,5.725904083251953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,4,128,1,fp8,fp8,0,5.947238540649414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,4,128,1,float16,float16,0,6.973075103759766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,8,128,1,float16,float16,0,6.9169151306152346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,8,128,1,float16,fp8,0,5.919084930419922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,1,128,1,float16,float16,0,3.481732940673828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,1,128,1,float16,fp8,0,2.940550422668457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,96,128,1,float16,float16,0,4.566638565063476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,96,128,1,float16,fp8,0,4.310902404785156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,96,128,1,fp8,fp8,0,4.04693603515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,1,128,1,fp8,fp8,0,2.9588703155517577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,8,128,1,fp8,fp8,0,5.876737594604492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,2,128,1,float16,float16,0,3.3017711639404297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,2,128,1,fp8,fp8,0,2.8315200805664062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,2,128,1,float16,fp8,0,3.0067663192749023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,4,128,1,fp8,fp8,0,2.8572240829467774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,4,128,1,float16,fp8,0,3.3057296752929686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,4,128,1,float16,float16,0,3.5699790954589843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,8,128,1,float16,float16,0,3.359969711303711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,8,128,1,float16,fp8,0,2.9203184127807615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,96,128,1,float16,float16,0,2.3106224060058596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,96,128,1,float16,fp8,0,2.080201530456543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,96,128,1,fp8,fp8,0,2.0467199325561523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,1,128,1,float16,fp8,0,1.5662528038024903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,1,128,1,float16,float16,0,2.031977653503418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,1,128,1,fp8,fp8,0,1.634307289123535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,2,128,1,float16,float16,0,1.5545712471008302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,8,128,1,fp8,fp8,0,3.1728912353515626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,2,128,1,float16,fp8,0,1.4253215789794922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,2,128,1,fp8,fp8,0,1.4243056297302246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,4,128,1,float16,float16,0,1.5581439971923827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,4,128,1,float16,fp8,0,1.5182000160217286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,4,128,1,fp8,fp8,0,1.4470751762390137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,8,128,1,float16,float16,0,1.6656431198120116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,8,128,1,float16,fp8,0,1.5156671524047851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,8,128,1,fp8,fp8,0,1.4818991661071776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,1,128,1,float16,fp8,0,21.23645782470703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,2,128,1,float16,fp8,0,21.5236083984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,2,128,1,fp8,fp8,0,21.69523468017578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,1,128,1,fp8,fp8,0,22.501341247558592
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,1,128,1,float16,float16,0,25.52098388671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,2,128,1,float16,float16,0,25.467245483398436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,4,128,1,float16,float16,0,26.56146240234375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,4,128,1,float16,fp8,0,21.53319396972656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,1,128,1,float16,float16,0,13.386192321777344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,96,128,1,float16,fp8,0,17.235188293457032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,4,128,1,fp8,fp8,0,22.808338928222657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,96,128,1,float16,float16,0,19.198846435546876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,96,128,1,fp8,fp8,0,17.28582305908203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,8,128,1,float16,fp8,0,22.964251708984374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,8,128,1,fp8,fp8,0,23.66376953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,8,128,1,float16,float16,0,26.209078979492187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,1,128,1,fp8,fp8,0,10.539923095703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,1,128,1,float16,fp8,0,10.890819549560547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,2,128,1,float16,fp8,0,10.680072021484374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,2,128,1,fp8,fp8,0,10.953411102294922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,2,128,1,float16,float16,0,12.610298919677735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,4,128,1,float16,fp8,0,11.257695770263672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,4,128,1,float16,float16,0,13.097511291503906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,4,128,1,fp8,fp8,0,10.868809509277344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,1,128,1,float16,float16,0,6.243732833862305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,1,128,1,float16,fp8,0,5.29967041015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,8,128,1,float16,fp8,0,11.080538940429687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,96,128,1,float16,fp8,0,8.596141052246093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,96,128,1,float16,float16,0,10.111529541015624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,96,128,1,fp8,fp8,0,8.749435424804688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,8,128,1,fp8,fp8,0,11.619681549072265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,8,128,1,float16,float16,0,13.682722473144532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,1,128,1,fp8,fp8,0,5.432057571411133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,2,128,1,float16,fp8,0,5.32384147644043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,2,128,1,float16,float16,0,6.318718338012696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,2,128,1,fp8,fp8,0,5.389487838745117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,4,128,1,float16,fp8,0,5.708547210693359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,4,128,1,fp8,fp8,0,5.412212753295899
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,4,128,1,float16,float16,0,6.729310607910156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,8,128,1,float16,float16,0,6.669827270507812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,1,128,1,float16,float16,0,2.9856815338134766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,8,128,1,float16,fp8,0,5.601448059082031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,1,128,1,float16,fp8,0,2.719553565979004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,96,128,1,float16,fp8,0,4.326132965087891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,96,128,1,float16,float16,0,4.850553512573242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,96,128,1,fp8,fp8,0,4.345054244995117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,1,128,1,fp8,fp8,0,2.771544075012207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,8,128,1,fp8,fp8,0,5.535484695434571
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,2,128,1,float16,fp8,0,2.679697608947754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,2,128,1,fp8,fp8,0,2.6885583877563475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,2,128,1,float16,float16,0,3.3210865020751954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,4,128,1,float16,fp8,0,2.7188751220703127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,4,128,1,float16,float16,0,3.0006687164306642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,4,128,1,fp8,fp8,0,2.7744384765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,8,128,1,float16,fp8,0,2.8785072326660157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,8,128,1,float16,float16,0,3.113150405883789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,1,128,1,float16,float16,0,1.5656448364257813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,1,128,1,float16,fp8,0,1.353001594543457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,96,128,1,float16,float16,0,2.359065628051758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,96,128,1,float16,fp8,0,2.305436706542969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,96,128,1,fp8,fp8,0,2.224836730957031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,8,128,1,fp8,fp8,0,2.7867919921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,1,128,1,fp8,fp8,0,1.3812432289123535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,2,128,1,float16,float16,0,1.5380640029907227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,2,128,1,float16,fp8,0,1.3837663650512695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,2,128,1,fp8,fp8,0,1.3539199829101562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,4,128,1,float16,float16,0,1.4996591567993165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,4,128,1,float16,fp8,0,1.373033618927002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,4,128,1,fp8,fp8,0,1.3978992462158204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,8,128,1,float16,float16,0,1.4978879928588866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,8,128,1,float16,fp8,0,1.4073087692260742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,8,128,1,fp8,fp8,0,1.4638287544250488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,96,128,1,float16,float16,0,1.1736031532287599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,1,128,1,float16,float16,0,0.7209792137145996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,96,128,1,float16,fp8,0,1.1027199745178222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,1,128,1,float16,fp8,0,0.6849535942077637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,96,128,1,fp8,fp8,0,1.1369296073913575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,1,128,1,fp8,fp8,0,0.7332111835479737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,2,128,1,float16,float16,0,0.7301807880401612
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,2,128,1,float16,fp8,0,0.7109024047851562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,2,128,1,fp8,fp8,0,0.6937871932983398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,4,128,1,float16,float16,0,0.7564191818237305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,4,128,1,float16,fp8,0,0.6976096153259277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,4,128,1,fp8,fp8,0,0.6999343872070313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,8,128,1,float16,float16,0,0.7701039791107178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,8,128,1,float16,fp8,0,0.7768864154815673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,8,128,1,fp8,fp8,0,0.7666656017303467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,1,128,1,float16,fp8,0,12.510964965820312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,1,128,1,fp8,fp8,0,12.552867126464843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,2,128,1,float16,fp8,0,12.652953338623046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,2,128,1,fp8,fp8,0,12.637052917480469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,1,128,1,float16,float16,0,14.671630859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,2,128,1,float16,float16,0,15.286585998535156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,4,128,1,float16,float16,0,14.953594970703126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,4,128,1,fp8,fp8,0,12.856552124023438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,4,128,1,float16,fp8,0,13.101513671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,8,128,1,float16,fp8,0,13.527528381347656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,96,128,1,float16,fp8,0,11.264396667480469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,96,128,1,fp8,fp8,0,11.334041595458984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,8,128,1,fp8,fp8,0,13.462765502929688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,96,128,1,float16,float16,0,12.395230102539063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,8,128,1,float16,float16,0,15.413661193847656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,1,128,1,float16,fp8,0,6.495510101318359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,1,128,1,float16,float16,0,7.929046630859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,1,128,1,fp8,fp8,0,6.2950927734375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,2,128,1,float16,fp8,0,6.4092353820800785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,2,128,1,fp8,fp8,0,6.721218872070312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,2,128,1,float16,float16,0,8.00772933959961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,4,128,1,float16,fp8,0,6.722367858886718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,4,128,1,float16,float16,0,7.914585876464844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,4,128,1,fp8,fp8,0,6.693803405761718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,1,128,1,float16,float16,0,3.6941375732421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,96,128,1,float16,float16,0,6.123894500732422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,8,128,1,float16,fp8,0,7.027703857421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,8,128,1,fp8,fp8,0,6.706362915039063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,96,128,1,float16,fp8,0,5.714030456542969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,96,128,1,fp8,fp8,0,5.6475166320800785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,8,128,1,float16,float16,0,7.956168365478516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,1,128,1,float16,fp8,0,3.1395055770874025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,1,128,1,fp8,fp8,0,3.3126049041748047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,2,128,1,float16,fp8,0,3.1715904235839845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,2,128,1,fp8,fp8,0,3.1792320251464843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,4,128,1,float16,fp8,0,3.215760040283203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,2,128,1,float16,float16,0,3.7006767272949217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,4,128,1,float16,float16,0,3.7375728607177736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,4,128,1,fp8,fp8,0,3.2346847534179686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,8,128,1,float16,float16,0,3.885348892211914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,8,128,1,float16,fp8,0,3.592420959472656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,1,128,1,float16,fp8,0,1.5866687774658204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,1,128,1,float16,float16,0,1.8242496490478515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,96,128,1,float16,fp8,0,2.8226608276367187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,96,128,1,float16,float16,0,3.0696144104003906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,96,128,1,fp8,fp8,0,2.8997663497924804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,8,128,1,fp8,fp8,0,3.325779342651367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,1,128,1,fp8,fp8,0,1.7084144592285155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,2,128,1,float16,fp8,0,1.5979855537414551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,2,128,1,fp8,fp8,0,1.6079103469848632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,2,128,1,float16,float16,0,2.1059087753295898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,4,128,1,float16,float16,0,1.7743663787841797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,4,128,1,fp8,fp8,0,1.6277536392211913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,4,128,1,float16,fp8,0,1.7175632476806642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,8,128,1,float16,float16,0,1.7964431762695312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,8,128,1,float16,fp8,0,1.7630559921264648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,8,128,1,fp8,fp8,0,1.6828943252563477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,1,128,1,float16,float16,0,0.8511072158813476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,96,128,1,float16,float16,0,1.5184752464294433
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,1,128,1,float16,fp8,0,0.8279536247253418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,96,128,1,float16,fp8,0,1.755963134765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,1,128,1,fp8,fp8,0,0.8382575988769532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,96,128,1,fp8,fp8,0,1.543017578125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,2,128,1,float16,float16,0,0.8703840255737305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,2,128,1,float16,fp8,0,0.8268799781799316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,2,128,1,fp8,fp8,0,0.8137984275817871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,4,128,1,float16,fp8,0,0.8456944465637207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,4,128,1,float16,float16,0,0.892801570892334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,4,128,1,fp8,fp8,0,0.8590640068054199
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,8,128,1,float16,float16,0,0.8943344116210937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,8,128,1,float16,fp8,0,0.8940400123596192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,8,128,1,fp8,fp8,0,0.8820240020751953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,96,128,1,float16,float16,0,0.785323190689087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,1,128,1,float16,float16,0,0.44570560455322267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,96,128,1,float16,fp8,0,0.7676239967346191
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,1,128,1,float16,fp8,0,0.4205647945404053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,96,128,1,fp8,fp8,0,0.7482048034667969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,1,128,1,fp8,fp8,0,0.4316671848297119
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,4,128,1,float16,float16,0,0.4567999839782715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,2,128,1,float16,float16,0,0.44979519844055177
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,2,128,1,float16,fp8,0,0.42635040283203124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,2,128,1,fp8,fp8,0,0.4274288177490234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,4,128,1,float16,fp8,0,0.44460320472717285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,4,128,1,fp8,fp8,0,0.4290736198425293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,8,128,1,float16,float16,0,0.4733407974243164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,8,128,1,float16,fp8,0,0.4621744155883789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,8,128,1,fp8,fp8,0,0.4495071887969971
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,1,128,1,fp8,fp8,0,12.334449768066406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,1,128,1,float16,fp8,0,12.33634262084961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,2,128,1,float16,fp8,0,12.800225830078125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,2,128,1,fp8,fp8,0,12.555048370361328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,1,128,1,float16,float16,0,13.847157287597657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,4,128,1,float16,fp8,0,12.799575805664062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,2,128,1,float16,float16,0,14.128877258300781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,4,128,1,float16,float16,0,14.470906066894532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,1,128,1,float16,float16,0,7.220851135253906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,4,128,1,fp8,fp8,0,12.780563354492188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,8,128,1,fp8,fp8,0,13.359422302246093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,96,128,1,float16,fp8,0,12.78439712524414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,96,128,1,float16,float16,0,13.611439514160157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,96,128,1,fp8,fp8,0,12.800927734375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,8,128,1,float16,fp8,0,14.300497436523438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,8,128,1,float16,float16,0,15.305201721191406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,1,128,1,float16,fp8,0,6.185031890869141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,1,128,1,fp8,fp8,0,6.2052257537841795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,2,128,1,fp8,fp8,0,6.263630294799805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,2,128,1,float16,fp8,0,6.392123031616211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,2,128,1,float16,float16,0,7.281267547607422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,4,128,1,float16,fp8,0,6.493927764892578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,4,128,1,float16,float16,0,7.978529357910157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,4,128,1,fp8,fp8,0,6.539921569824219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,1,128,1,float16,float16,0,3.631447982788086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,1,128,1,float16,fp8,0,3.2171710968017577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,8,128,1,float16,float16,0,8.630595397949218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,8,128,1,fp8,fp8,0,6.683753967285156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,8,128,1,float16,fp8,0,6.699568176269532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,96,128,1,float16,float16,0,6.775532531738281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,96,128,1,float16,fp8,0,6.457324981689453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,96,128,1,fp8,fp8,0,6.412171173095703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,1,128,1,fp8,fp8,0,3.129377555847168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,2,128,1,float16,float16,0,3.5251232147216798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,2,128,1,float16,fp8,0,3.172599983215332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,2,128,1,fp8,fp8,0,3.1337888717651365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,4,128,1,fp8,fp8,0,3.207223892211914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,4,128,1,float16,fp8,0,3.2586959838867187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,4,128,1,float16,float16,0,3.6957969665527344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,8,128,1,float16,float16,0,3.840318298339844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,8,128,1,float16,fp8,0,3.5865665435791017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,1,128,1,float16,float16,0,1.6448816299438476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,1,128,1,float16,fp8,0,1.668452835083008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,8,128,1,fp8,fp8,0,3.587548828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,1,128,1,fp8,fp8,0,1.708510398864746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,96,128,1,float16,float16,0,3.4056064605712892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,96,128,1,float16,fp8,0,3.214166259765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,96,128,1,fp8,fp8,0,3.1979984283447265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,2,128,1,float16,fp8,0,1.626523208618164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,2,128,1,fp8,fp8,0,1.5871343612670898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,2,128,1,float16,float16,0,2.044241523742676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,4,128,1,float16,float16,0,1.7565872192382812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,4,128,1,float16,fp8,0,1.652110481262207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,4,128,1,fp8,fp8,0,1.6478784561157227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,8,128,1,float16,float16,0,1.826683235168457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,8,128,1,float16,fp8,0,1.6738576889038086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,1,128,1,float16,float16,0,0.8308976173400879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,1,128,1,float16,fp8,0,0.8124431610107422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,8,128,1,fp8,fp8,0,1.7267648696899414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,96,128,1,float16,float16,0,1.719852828979492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,1,128,1,fp8,fp8,0,0.8380767822265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,96,128,1,float16,fp8,0,1.7621040344238281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,96,128,1,fp8,fp8,0,1.7309728622436524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,2,128,1,float16,float16,0,0.8440095901489257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,2,128,1,float16,fp8,0,0.816708755493164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,2,128,1,fp8,fp8,0,0.8184240341186524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,4,128,1,float16,float16,0,0.8744095802307129
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,4,128,1,float16,fp8,0,0.8433263778686524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,4,128,1,fp8,fp8,0,0.8477616310119629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,8,128,1,float16,float16,0,0.9168959617614746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,8,128,1,float16,fp8,0,0.8820528030395508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,8,128,1,fp8,fp8,0,0.8782799720764161
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,96,128,1,float16,float16,0,0.8790464401245117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,1,128,1,float16,float16,0,0.4426623821258545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,96,128,1,float16,fp8,0,0.8478832244873047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,1,128,1,float16,fp8,0,0.4213088035583496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,1,128,1,fp8,fp8,0,0.4199696063995361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,96,128,1,fp8,fp8,0,0.8524031639099121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,2,128,1,float16,fp8,0,0.4257952213287354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,2,128,1,float16,float16,0,0.4458928108215332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,2,128,1,fp8,fp8,0,0.42531681060791016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,4,128,1,float16,float16,0,0.456328010559082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,4,128,1,float16,fp8,0,0.4319632053375244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,4,128,1,fp8,fp8,0,0.4344655990600586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,8,128,1,float16,float16,0,0.46943840980529783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,8,128,1,float16,fp8,0,0.4544032096862793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,8,128,1,fp8,fp8,0,0.4515888214111328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,96,128,1,float16,float16,0,0.45537600517272947
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,96,128,1,float16,fp8,0,0.43866720199584963
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,1,128,1,float16,float16,0,0.23438079357147218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,96,128,1,fp8,fp8,0,0.4374095916748047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,1,128,1,float16,fp8,0,0.2238624095916748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,1,128,1,fp8,fp8,0,0.2232815980911255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,2,128,1,float16,float16,0,0.2361135959625244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,2,128,1,float16,fp8,0,0.2257312059402466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,2,128,1,fp8,fp8,0,0.22469758987426758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,4,128,1,float16,float16,0,0.24104321002960205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,4,128,1,float16,fp8,0,0.2315135955810547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,4,128,1,fp8,fp8,0,0.23096160888671874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,8,128,1,float16,float16,0,0.25036959648132323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,8,128,1,float16,fp8,0,0.2377295970916748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,8,128,1,fp8,fp8,0,0.23895039558410644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,1,128,1,float16,fp8,0,7.714673614501953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,1,128,1,fp8,fp8,0,7.69041748046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,1,128,1,float16,float16,0,8.428427124023438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,2,128,1,float16,fp8,0,7.844000244140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,2,128,1,fp8,fp8,0,7.817353820800781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,2,128,1,float16,float16,0,8.600141143798828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,4,128,1,float16,fp8,0,7.988275146484375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,4,128,1,float16,float16,0,8.918313598632812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,1,128,1,float16,float16,0,4.361091232299804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,4,128,1,fp8,fp8,0,7.975543975830078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,8,128,1,float16,fp8,0,8.3734130859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,8,128,1,fp8,fp8,0,8.395875549316406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,8,128,1,float16,float16,0,9.269159698486328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,96,128,1,float16,float16,0,9.331584167480468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,96,128,1,float16,fp8,0,8.878775787353515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,96,128,1,fp8,fp8,0,8.79622573852539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,1,128,1,float16,fp8,0,3.8294559478759767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,1,128,1,fp8,fp8,0,3.8176910400390627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,2,128,1,float16,float16,0,4.330352020263672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,2,128,1,float16,fp8,0,3.915332794189453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,2,128,1,fp8,fp8,0,3.961703872680664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,4,128,1,float16,fp8,0,3.9750465393066405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,4,128,1,float16,float16,0,4.261177444458008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,4,128,1,fp8,fp8,0,3.995159912109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,1,128,1,float16,float16,0,2.037291145324707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,8,128,1,float16,float16,0,4.737216186523438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,8,128,1,float16,fp8,0,4.223982238769532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,8,128,1,fp8,fp8,0,4.1734577178955075
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,1,128,1,float16,fp8,0,1.9246576309204102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,96,128,1,float16,fp8,0,4.4111583709716795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,96,128,1,float16,float16,0,4.679518508911133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,96,128,1,fp8,fp8,0,4.59954719543457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,2,128,1,fp8,fp8,0,1.9444704055786133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,1,128,1,fp8,fp8,0,1.9668256759643554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,2,128,1,float16,float16,0,2.0250528335571287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,2,128,1,float16,fp8,0,1.9558399200439454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,4,128,1,float16,float16,0,2.1217344284057615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,4,128,1,float16,fp8,0,2.0090864181518553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,4,128,1,fp8,fp8,0,2.045947265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,8,128,1,float16,float16,0,2.259974479675293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,1,128,1,float16,float16,0,1.003001594543457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,1,128,1,float16,fp8,0,0.993342399597168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,1,128,1,fp8,fp8,0,1.0270288467407227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,8,128,1,float16,fp8,0,2.097127914428711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,8,128,1,fp8,fp8,0,2.1206335067749023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,96,128,1,float16,float16,0,2.3630544662475588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,96,128,1,float16,fp8,0,2.2286895751953124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,96,128,1,fp8,fp8,0,2.372865676879883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,2,128,1,float16,fp8,0,0.9985407829284668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,2,128,1,float16,float16,0,1.0718640327453612
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,2,128,1,fp8,fp8,0,1.0143280029296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,4,128,1,float16,float16,0,1.0485695838928222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,4,128,1,float16,fp8,0,1.0195584297180176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,4,128,1,fp8,fp8,0,1.0394207954406738
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,8,128,1,float16,float16,0,1.117636775970459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,8,128,1,float16,fp8,0,1.077139186859131
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,8,128,1,fp8,fp8,0,1.0700655937194825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,1,128,1,float16,float16,0,0.514793586730957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,1,128,1,float16,fp8,0,0.5096848011016846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,96,128,1,float16,float16,0,1.2074591636657714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,96,128,1,float16,fp8,0,1.1588624000549317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,96,128,1,fp8,fp8,0,1.1696911811828614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,1,128,1,fp8,fp8,0,0.5266575813293457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,2,128,1,float16,float16,0,0.5244783878326416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,2,128,1,float16,fp8,0,0.5097871780395508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,2,128,1,fp8,fp8,0,0.5092175960540771
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,4,128,1,float16,float16,0,0.5481808185577393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,4,128,1,float16,fp8,0,0.5339392185211181
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,4,128,1,fp8,fp8,0,0.5305359840393067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,8,128,1,float16,float16,0,0.5984752178192139
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,96,128,1,float16,float16,0,0.5988560199737549
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,8,128,1,float16,fp8,0,0.5612495899200439
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,8,128,1,fp8,fp8,0,0.5493040084838867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,96,128,1,float16,fp8,0,0.5811840057373047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,1,128,1,float16,float16,0,0.2780447959899902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,96,128,1,fp8,fp8,0,0.5950384140014648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,1,128,1,float16,fp8,0,0.267737603187561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,1,128,1,fp8,fp8,0,0.2723999977111816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,2,128,1,float16,float16,0,0.2785248041152954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,2,128,1,float16,fp8,0,0.27142720222473143
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,2,128,1,fp8,fp8,0,0.26676321029663086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,4,128,1,float16,float16,0,0.2863503932952881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,4,128,1,float16,fp8,0,0.27836639881134034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,4,128,1,fp8,fp8,0,0.27765119075775146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,8,128,1,float16,float16,0,0.30152320861816406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,8,128,1,float16,fp8,0,0.2937999963760376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,8,128,1,fp8,fp8,0,0.29085440635681153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,96,128,1,float16,float16,0,0.322545599937439
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,96,128,1,float16,fp8,0,0.31059999465942384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,1,128,1,float16,float16,0,0.15313279628753662
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,96,128,1,fp8,fp8,0,0.31084160804748534
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,4,128,1,float16,float16,0,0.1555456042289734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,1,128,1,float16,fp8,0,0.1483024001121521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,1,128,1,fp8,fp8,0,0.14891519546508789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,2,128,1,float16,float16,0,0.15380799770355225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,2,128,1,float16,fp8,0,0.14852479696273804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,2,128,1,fp8,fp8,0,0.14875999689102173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,4,128,1,float16,fp8,0,0.15141279697418214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,4,128,1,fp8,fp8,0,0.15057920217514037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,8,128,1,float16,float16,0,0.1635151982307434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,8,128,1,float16,fp8,0,0.15793440341949463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,8,128,1,fp8,fp8,0,0.15805599689483643
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,1,128,1,float16,fp8,0,8.092935943603516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,1,128,1,fp8,fp8,0,8.07994384765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,1,128,1,float16,float16,0,8.557119750976563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,2,128,1,float16,fp8,0,8.231470489501953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,2,128,1,fp8,fp8,0,8.240734100341797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,2,128,1,float16,float16,0,8.6748291015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,4,128,1,float16,fp8,0,8.532268524169922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,4,128,1,float16,float16,0,8.986265563964844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,4,128,1,fp8,fp8,0,8.528855895996093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,8,128,1,float16,fp8,0,8.98996810913086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,8,128,1,fp8,fp8,0,9.028179168701172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,8,128,1,float16,float16,0,9.918192291259766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,96,128,1,float16,fp8,0,10.745321655273438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,96,128,1,float16,float16,0,11.252545928955078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,96,128,1,fp8,fp8,0,10.755123138427734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,1,128,1,float16,float16,0,4.229905700683593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,1,128,1,float16,fp8,0,4.0119071960449215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,1,128,1,fp8,fp8,0,4.0316017150878904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,2,128,1,float16,float16,0,4.365316772460938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,2,128,1,float16,fp8,0,4.1342414855957035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,2,128,1,fp8,fp8,0,4.127334213256836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,4,128,1,float16,float16,0,4.562547302246093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,4,128,1,float16,fp8,0,4.241342544555664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,4,128,1,fp8,fp8,0,4.343337631225586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,8,128,1,float16,float16,0,4.791435241699219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,8,128,1,float16,fp8,0,4.5252433776855465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,1,128,1,float16,float16,0,2.0700544357299804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,8,128,1,fp8,fp8,0,4.540028762817383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,1,128,1,float16,fp8,0,2.0373903274536134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,96,128,1,float16,float16,0,5.633748626708984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,1,128,1,fp8,fp8,0,2.043435287475586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,2,128,1,float16,float16,0,2.1662927627563477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,2,128,1,float16,fp8,0,2.1023168563842773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,96,128,1,float16,fp8,0,5.390881729125977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,2,128,1,fp8,fp8,0,2.0688192367553713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,96,128,1,fp8,fp8,0,5.392787170410156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,4,128,1,float16,float16,0,2.1914655685424806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,4,128,1,float16,fp8,0,2.162411117553711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,4,128,1,fp8,fp8,0,2.1263120651245115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,8,128,1,float16,float16,0,2.4669008255004883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,8,128,1,float16,fp8,0,2.2633119583129884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,8,128,1,fp8,fp8,0,2.2737632751464845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,1,128,1,float16,float16,0,1.0474703788757325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,1,128,1,float16,fp8,0,1.0209983825683593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,96,128,1,float16,float16,0,2.7995344161987306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,96,128,1,float16,fp8,0,2.7075056076049804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,1,128,1,fp8,fp8,0,1.0812368392944336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,2,128,1,float16,float16,0,1.0518735885620116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,96,128,1,fp8,fp8,0,2.789190483093262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,2,128,1,float16,fp8,0,1.0457615852355957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,2,128,1,fp8,fp8,0,1.0499855995178222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,4,128,1,float16,float16,0,1.0892255783081055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,4,128,1,float16,fp8,0,1.0792112350463867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,4,128,1,fp8,fp8,0,1.0839391708374024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,8,128,1,float16,float16,0,1.2160271644592284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,8,128,1,float16,fp8,0,1.152177619934082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,8,128,1,fp8,fp8,0,1.17053279876709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,1,128,1,float16,float16,0,0.5336351871490479
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,96,128,1,float16,float16,0,1.4072959899902344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,1,128,1,float16,fp8,0,0.5276319980621338
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,96,128,1,float16,fp8,0,1.3667984008789062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,2,128,1,float16,fp8,0,0.5374192237854004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,2,128,1,float16,float16,0,0.5408112049102783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,1,128,1,fp8,fp8,0,0.5282864093780517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,96,128,1,fp8,fp8,0,1.383407974243164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,2,128,1,fp8,fp8,0,0.5345424175262451
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,4,128,1,float16,float16,0,0.5718768119812012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,4,128,1,float16,fp8,0,0.5542175769805908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,4,128,1,fp8,fp8,0,0.5586976051330567
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,8,128,1,float16,float16,0,0.6018479824066162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,8,128,1,float16,fp8,0,0.5917759895324707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,8,128,1,fp8,fp8,0,0.587879991531372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,1,128,1,float16,fp8,0,0.2780976057052612
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,96,128,1,fp8,fp8,0,0.7088240146636963
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,96,128,1,float16,float16,0,0.7206768035888672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,1,128,1,float16,float16,0,0.2762048006057739
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,96,128,1,float16,fp8,0,0.7093183994293213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,1,128,1,fp8,fp8,0,0.27634079456329347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,2,128,1,float16,float16,0,0.2903903961181641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,2,128,1,float16,fp8,0,0.2809808015823364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,2,128,1,fp8,fp8,0,0.2838655948638916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,4,128,1,float16,float16,0,0.29792799949646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,4,128,1,float16,fp8,0,0.29475200176239014
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,4,128,1,fp8,fp8,0,0.2861648082733154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,8,128,1,float16,float16,0,0.3183568000793457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,8,128,1,float16,fp8,0,0.3079200029373169
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,8,128,1,fp8,fp8,0,0.3117408037185669
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,96,128,1,float16,float16,0,0.37437119483947756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,96,128,1,float16,fp8,0,0.3675584077835083
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,1,128,1,float16,float16,0,0.15575040578842164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,96,128,1,fp8,fp8,0,0.36757760047912597
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,4,128,1,float16,float16,0,0.16123679876327515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,1,128,1,float16,fp8,0,0.15259679555892944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,1,128,1,fp8,fp8,0,0.15266560316085814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,2,128,1,float16,float16,0,0.15804320573806763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,2,128,1,float16,fp8,0,0.15445280075073242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,8,128,1,fp8,fp8,0,0.16657919883728028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,2,128,1,fp8,fp8,0,0.15090240240097047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,4,128,1,float16,fp8,0,0.16015360355377198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,4,128,1,fp8,fp8,0,0.15877599716186525
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,8,128,1,float16,float16,0,0.1694175958633423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,8,128,1,float16,fp8,0,0.16724319458007814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,96,128,1,float16,float16,0,0.20563199520111083
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,96,128,1,float16,fp8,0,0.19649280309677125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,96,128,1,fp8,fp8,0,0.19606080055236816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,1,128,1,float16,float16,0,0.08993759751319885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,4,128,1,float16,float16,0,0.09251520037651062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,4,128,1,float16,fp8,0,0.08888319730758668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,1,128,1,float16,fp8,0,0.0886847972869873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,1,128,1,fp8,fp8,0,0.08761600255966187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,2,128,1,float16,float16,0,0.09192960262298584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,2,128,1,float16,fp8,0,0.08872640132904053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,2,128,1,fp8,fp8,0,0.08900319933891296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,4,128,1,fp8,fp8,0,0.09019839763641357
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,8,128,1,float16,float16,0,0.09839040040969849
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,8,128,1,float16,fp8,0,0.09619839787483216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,8,128,1,fp8,fp8,0,0.09463199973106384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,1,128,1,float16,float16,0,5.879679870605469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,1,128,1,float16,fp8,0,5.833689498901367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,1,128,1,fp8,fp8,0,5.832803344726562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,2,128,1,float16,float16,0,6.054913711547852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,2,128,1,float16,fp8,0,5.9957328796386715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,2,128,1,fp8,fp8,0,6.075142288208008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,4,128,1,float16,float16,0,6.319214248657227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,4,128,1,fp8,fp8,0,6.238969421386718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,4,128,1,float16,fp8,0,6.2489887237548825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,8,128,1,float16,float16,0,6.928028869628906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,8,128,1,float16,fp8,0,6.833604431152343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,8,128,1,fp8,fp8,0,6.823255920410157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,1,128,1,float16,float16,0,2.9168703079223635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,1,128,1,float16,fp8,0,2.912692832946777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,96,128,1,float16,float16,0,10.030315399169922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,96,128,1,float16,fp8,0,9.760985565185546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,1,128,1,fp8,fp8,0,2.925649642944336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,2,128,1,float16,float16,0,3.002566337585449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,2,128,1,float16,fp8,0,2.9979999542236326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,96,128,1,fp8,fp8,0,9.826308441162109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,2,128,1,fp8,fp8,0,2.9853919982910155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,4,128,1,float16,float16,0,3.14890079498291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,4,128,1,float16,fp8,0,3.1578832626342774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,4,128,1,fp8,fp8,0,3.149687957763672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,8,128,1,float16,float16,0,3.47203369140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,8,128,1,float16,fp8,0,3.416350555419922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,8,128,1,fp8,fp8,0,3.4328704833984376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,1,128,1,float16,float16,0,1.4597200393676757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,1,128,1,float16,fp8,0,1.4737088203430175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,1,128,1,fp8,fp8,0,1.4755279541015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,2,128,1,float16,float16,0,1.50274715423584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,2,128,1,float16,fp8,0,1.5110735893249512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,96,128,1,float16,float16,0,4.985691070556641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,2,128,1,fp8,fp8,0,1.510747241973877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,96,128,1,float16,fp8,0,4.8355857849121096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,4,128,1,float16,float16,0,1.5756095886230468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,4,128,1,float16,fp8,0,1.5859984397888183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,96,128,1,fp8,fp8,0,4.862847900390625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,4,128,1,fp8,fp8,0,1.5767600059509277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,8,128,1,float16,float16,0,1.7441200256347655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,8,128,1,float16,fp8,0,1.7295072555541993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,1,128,1,float16,float16,0,0.7469168186187745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,1,128,1,float16,fp8,0,0.7449984073638916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,8,128,1,fp8,fp8,0,1.7100671768188476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,1,128,1,fp8,fp8,0,0.7488976001739502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,2,128,1,float16,float16,0,0.7663824081420898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,96,128,1,float16,float16,0,2.499193572998047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,2,128,1,float16,fp8,0,0.7689311981201172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,96,128,1,float16,fp8,0,2.42795352935791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,2,128,1,fp8,fp8,0,0.7644832134246826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,96,128,1,fp8,fp8,0,2.4509328842163085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,4,128,1,float16,float16,0,0.7992656230926514
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,4,128,1,float16,fp8,0,0.8041551589965821
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,4,128,1,fp8,fp8,0,0.8053232192993164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,8,128,1,float16,fp8,0,0.87467041015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,8,128,1,float16,float16,0,0.9014127731323243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,8,128,1,fp8,fp8,0,0.8675488471984864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,1,128,1,float16,float16,0,0.386516809463501
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,1,128,1,float16,fp8,0,0.3856192111968994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,1,128,1,fp8,fp8,0,0.3884671926498413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,96,128,1,float16,float16,0,1.2595456123352051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,96,128,1,float16,fp8,0,1.237508773803711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,2,128,1,float16,float16,0,0.4008624076843262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,96,128,1,fp8,fp8,0,1.227228832244873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,2,128,1,float16,fp8,0,0.393123197555542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,2,128,1,fp8,fp8,0,0.3959088087081909
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,4,128,1,float16,float16,0,0.4119887828826904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,4,128,1,float16,fp8,0,0.41464958190917967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,4,128,1,fp8,fp8,0,0.41276960372924804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,8,128,1,float16,float16,0,0.4536719799041748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,8,128,1,float16,fp8,0,0.45669121742248536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,8,128,1,fp8,fp8,0,0.44544157981872556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,96,128,1,float16,float16,0,0.6405856132507324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,96,128,1,float16,fp8,0,0.6317535877227783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,1,128,1,float16,float16,0,0.2041088104248047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,1,128,1,float16,fp8,0,0.20893759727478028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,96,128,1,fp8,fp8,0,0.6286143779754638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,1,128,1,fp8,fp8,0,0.20546560287475585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,2,128,1,float16,float16,0,0.21392478942871093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,2,128,1,float16,fp8,0,0.20899999141693115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,2,128,1,fp8,fp8,0,0.20854721069335938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,4,128,1,float16,float16,0,0.21825120449066163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,4,128,1,float16,fp8,0,0.21852800846099854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,4,128,1,fp8,fp8,0,0.22328639030456543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,8,128,1,float16,float16,0,0.24239840507507324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,8,128,1,float16,fp8,0,0.23723359107971193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,8,128,1,fp8,fp8,0,0.2408992052078247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,96,128,1,float16,float16,0,0.33438398838043215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,96,128,1,float16,fp8,0,0.3248415946960449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,96,128,1,fp8,fp8,0,0.32809441089630126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,1,128,1,float16,float16,0,0.11485919952392579
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,1,128,1,float16,fp8,0,0.11828800439834594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,1,128,1,fp8,fp8,0,0.11780639886856079
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,2,128,1,float16,float16,0,0.11608799695968627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,4,128,1,fp8,fp8,0,0.12588000297546387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,8,128,1,float16,float16,0,0.13139359951019286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,2,128,1,float16,fp8,0,0.11895999908447266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,2,128,1,fp8,fp8,0,0.1167456030845642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,4,128,1,float16,float16,0,0.12177439928054809
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,4,128,1,float16,fp8,0,0.12268320322036744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,8,128,1,float16,fp8,0,0.13070080280303956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,8,128,1,fp8,fp8,0,0.1316799998283386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,96,128,1,float16,float16,0,0.1839727997779846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,96,128,1,float16,fp8,0,0.17857120037078858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,96,128,1,fp8,fp8,0,0.177892804145813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,1,128,1,float16,float16,0,0.07087839841842651
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,1,128,1,float16,fp8,0,0.06909599900245667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,1,128,1,fp8,fp8,0,0.0686016023159027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,2,128,1,float16,float16,0,0.07161440253257752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,2,128,1,float16,fp8,0,0.07000319957733155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,2,128,1,fp8,fp8,0,0.06973440051078797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,4,128,1,float16,float16,0,0.0715120017528534
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,4,128,1,float16,fp8,0,0.07084000110626221
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,4,128,1,fp8,fp8,0,0.07043840289115906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,96,128,1,float16,fp8,0,0.10026400089263916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,8,128,1,float16,float16,0,0.07768800258636474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,8,128,1,float16,fp8,0,0.077292799949646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,8,128,1,fp8,fp8,0,0.07767999768257142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,96,128,1,float16,float16,0,0.10360000133514405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,96,128,1,fp8,fp8,0,0.10033760070800782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,1,128,1,float16,float16,0,0.04036799967288971
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,1,128,1,float16,fp8,0,0.041289600729942325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,1,128,1,fp8,fp8,0,0.041863998770713805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,2,128,1,float16,float16,0,0.04145439863204956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,2,128,1,float16,fp8,0,0.0419840008020401
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,2,128,1,fp8,fp8,0,0.0418368011713028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,4,128,1,float16,float16,0,0.041222399473190306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,4,128,1,float16,fp8,0,0.04175359904766083
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,4,128,1,fp8,fp8,0,0.041631999611854556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,8,128,1,float16,float16,0,0.042931199073791504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,8,128,1,float16,fp8,0,0.04248799979686737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,8,128,1,fp8,fp8,0,0.04263199865818024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,1,128,1,float16,float16,0,2.298294448852539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,1,128,1,float16,fp8,0,2.4229408264160157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,1,128,1,fp8,fp8,0,2.4068336486816406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,2,128,1,float16,float16,0,2.377739143371582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,2,128,1,float16,fp8,0,2.4818944931030273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,2,128,1,fp8,fp8,0,2.4808319091796873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,4,128,1,float16,float16,0,2.540171241760254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,4,128,1,float16,fp8,0,2.637131118774414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,4,128,1,fp8,fp8,0,2.6267135620117186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,1,128,1,float16,float16,0,1.1665328025817872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,8,128,1,float16,float16,0,2.853803253173828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,8,128,1,float16,fp8,0,2.9034671783447266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,8,128,1,fp8,fp8,0,2.9016607284545897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,1,128,1,float16,fp8,0,1.2155983924865723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,1,128,1,fp8,fp8,0,1.216865634918213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,2,128,1,float16,float16,0,1.1924863815307618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,2,128,1,float16,fp8,0,1.2453743934631347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,2,128,1,fp8,fp8,0,1.237720012664795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,96,128,1,float16,float16,0,4.768569564819336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,96,128,1,float16,fp8,0,4.551623916625976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,96,128,1,fp8,fp8,0,4.571531295776367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,4,128,1,float16,float16,0,1.2813008308410645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,4,128,1,fp8,fp8,0,1.3113120079040528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,4,128,1,float16,fp8,0,1.326471996307373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,8,128,1,float16,fp8,0,1.448798370361328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,8,128,1,float16,float16,0,1.4336288452148438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,1,128,1,float16,float16,0,0.5870863914489746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,8,128,1,fp8,fp8,0,1.465835189819336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,1,128,1,float16,fp8,0,0.6187520027160645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,1,128,1,fp8,fp8,0,0.613036823272705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,2,128,1,float16,float16,0,0.6096608161926269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,2,128,1,float16,fp8,0,0.631763219833374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,96,128,1,float16,float16,0,2.3981199264526367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,2,128,1,fp8,fp8,0,0.6382224082946777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,96,128,1,float16,fp8,0,2.2949039459228517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,4,128,1,float16,float16,0,0.6504096031188965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,4,128,1,float16,fp8,0,0.6664576053619384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,96,128,1,fp8,fp8,0,2.2767744064331055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,4,128,1,fp8,fp8,0,0.6706751823425293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,8,128,1,float16,float16,0,0.7258111953735351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,8,128,1,fp8,fp8,0,0.739244794845581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,8,128,1,float16,fp8,0,0.743998384475708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,1,128,1,float16,float16,0,0.30870718955993653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,1,128,1,float16,fp8,0,0.32173919677734375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,96,128,1,float16,float16,0,1.2156975746154786
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,1,128,1,fp8,fp8,0,0.3197216033935547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,2,128,1,float16,float16,0,0.3165199995040894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,96,128,1,float16,fp8,0,1.1592063903808594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,96,128,1,fp8,fp8,0,1.1461536407470703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,2,128,1,float16,fp8,0,0.33151519298553467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,4,128,1,float16,fp8,0,0.3497920036315918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,2,128,1,fp8,fp8,0,0.33174240589141846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,4,128,1,float16,float16,0,0.33677599430084226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,4,128,1,fp8,fp8,0,0.3461967945098877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,8,128,1,float16,float16,0,0.375601601600647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,8,128,1,float16,fp8,0,0.38754398822784425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,8,128,1,fp8,fp8,0,0.3792448043823242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,1,128,1,float16,float16,0,0.16529760360717774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,96,128,1,float16,float16,0,0.6235936164855957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,96,128,1,float16,fp8,0,0.5936895847320557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,1,128,1,float16,fp8,0,0.17147840261459352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,96,128,1,fp8,fp8,0,0.5932896137237549
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,1,128,1,fp8,fp8,0,0.1718384027481079
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,2,128,1,float16,float16,0,0.17359999418258668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,2,128,1,float16,fp8,0,0.1766095995903015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,2,128,1,fp8,fp8,0,0.1786288022994995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,4,128,1,float16,float16,0,0.18058079481124878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,4,128,1,float16,fp8,0,0.18477920293807984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,4,128,1,fp8,fp8,0,0.18669120073318482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,8,128,1,float16,float16,0,0.2000432014465332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,8,128,1,float16,fp8,0,0.20678560733795165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,8,128,1,fp8,fp8,0,0.2019792079925537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,96,128,1,float16,float16,0,0.32482399940490725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,96,128,1,float16,fp8,0,0.31036159992218015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,96,128,1,fp8,fp8,0,0.30674560070037843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,1,128,1,float16,float16,0,0.09559199810028077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,1,128,1,float16,fp8,0,0.0990447998046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,4,128,1,float16,float16,0,0.1015936017036438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,1,128,1,fp8,fp8,0,0.1011072039604187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,2,128,1,float16,float16,0,0.09577919840812683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,2,128,1,float16,fp8,0,0.10043200254440307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,2,128,1,fp8,fp8,0,0.10079200267791748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,4,128,1,float16,fp8,0,0.10661920309066772
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,4,128,1,fp8,fp8,0,0.1060256004333496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,8,128,1,float16,float16,0,0.11080960035324097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,8,128,1,float16,fp8,0,0.11478879451751708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,8,128,1,fp8,fp8,0,0.11352640390396118
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,96,128,1,float16,float16,0,0.17698559761047364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,96,128,1,float16,fp8,0,0.16776800155639648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,96,128,1,fp8,fp8,0,0.1671504020690918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,1,128,1,float16,float16,0,0.061324799060821535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,1,128,1,float16,fp8,0,0.061587202548980716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,1,128,1,fp8,fp8,0,0.0613647997379303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,2,128,1,float16,float16,0,0.06132320165634155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,2,128,1,float16,fp8,0,0.06233599781990051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,2,128,1,fp8,fp8,0,0.06246560215950012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,4,128,1,float16,float16,0,0.0626255989074707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,4,128,1,float16,fp8,0,0.06297119855880737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,4,128,1,fp8,fp8,0,0.06277120113372803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,8,128,1,float16,float16,0,0.06894400119781494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,1,128,1,float16,float16,0,0.036057600378990175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,8,128,1,float16,fp8,0,0.06801440119743347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,1,128,1,float16,fp8,0,0.038124799728393555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,8,128,1,fp8,fp8,0,0.06955199837684631
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,96,128,1,float16,float16,0,0.1000048041343689
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,96,128,1,float16,fp8,0,0.09502559900283813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,96,128,1,fp8,fp8,0,0.09439679980278015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,1,128,1,fp8,fp8,0,0.03794879913330078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,2,128,1,float16,float16,0,0.0362960010766983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,2,128,1,float16,fp8,0,0.03783040046691895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,2,128,1,fp8,fp8,0,0.037959998846054076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,4,128,1,float16,float16,0,0.038104000687599185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,4,128,1,float16,fp8,0,0.03844000101089477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,4,128,1,fp8,fp8,0,0.03871839940547943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,8,128,1,float16,float16,0,0.03861280083656311
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,8,128,1,float16,fp8,0,0.038945600390434265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,8,128,1,fp8,fp8,0,0.03866879940032959
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,96,128,1,float16,float16,0,0.056446397304534913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,96,128,1,float16,fp8,0,0.04995999932289123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,96,128,1,fp8,fp8,0,0.05028799772262573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,1,128,1,float16,float16,0,0.028731200098991393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,1,128,1,float16,fp8,0,0.0284496009349823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,1,128,1,fp8,fp8,0,0.02781440019607544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,2,128,1,float16,float16,0,0.028540799021720888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,2,128,1,float16,fp8,0,0.028070399165153505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,2,128,1,fp8,fp8,0,0.028441599011421202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,4,128,1,float16,float16,0,0.02869440019130707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,4,128,1,float16,fp8,0,0.02847360074520111
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,4,128,1,fp8,fp8,0,0.0281792014837265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,8,128,1,float16,float16,0,0.02884800136089325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,8,128,1,float16,fp8,0,0.02855840027332306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,8,128,1,fp8,fp8,0,0.02871040105819702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,1,128,1,float16,float16,0,1.090772819519043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,1,128,1,float16,fp8,0,1.174300765991211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,1,128,1,fp8,fp8,0,1.1777359962463378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,2,128,1,float16,float16,0,1.1249168395996094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,2,128,1,float16,fp8,0,1.20861759185791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,2,128,1,fp8,fp8,0,1.213425636291504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,4,128,1,float16,float16,0,1.1994624137878418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,4,128,1,float16,fp8,0,1.2994367599487304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,4,128,1,fp8,fp8,0,1.291708755493164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,8,128,1,float16,float16,0,1.3477952003479003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,1,128,1,float16,float16,0,0.5559887886047363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,8,128,1,float16,fp8,0,1.4591055870056153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,1,128,1,float16,fp8,0,0.5985968112945557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,8,128,1,fp8,fp8,0,1.4529264450073243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,1,128,1,fp8,fp8,0,0.5981887817382813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,2,128,1,float16,float16,0,0.5722608089447021
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,2,128,1,float16,fp8,0,0.6202447891235352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,2,128,1,fp8,fp8,0,0.6190703868865967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,4,128,1,float16,float16,0,0.6077807903289795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,96,128,1,float16,float16,0,2.3928064346313476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,96,128,1,float16,fp8,0,2.3501455307006838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,96,128,1,fp8,fp8,0,2.337758445739746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,4,128,1,float16,fp8,0,0.6550816059112549
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,4,128,1,fp8,fp8,0,0.6565824031829834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,8,128,1,float16,float16,0,0.6821807861328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,8,128,1,float16,fp8,0,0.7340384006500245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,1,128,1,float16,float16,0,0.2904239892959595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,8,128,1,fp8,fp8,0,0.7352208137512207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,1,128,1,float16,fp8,0,0.3107167959213257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,1,128,1,fp8,fp8,0,0.3112175941467285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,96,128,1,float16,fp8,0,1.1815664291381835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,96,128,1,float16,float16,0,1.1989007949829102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,2,128,1,float16,float16,0,0.2981328010559082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,96,128,1,fp8,fp8,0,1.1808560371398926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,2,128,1,float16,fp8,0,0.3197648048400879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,2,128,1,fp8,fp8,0,0.32014880180358884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,4,128,1,float16,float16,0,0.31668479442596437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,4,128,1,float16,fp8,0,0.3380928039550781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,4,128,1,fp8,fp8,0,0.3378832101821899
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,8,128,1,float16,float16,0,0.3511615991592407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,8,128,1,float16,fp8,0,0.3800479888916016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,8,128,1,fp8,fp8,0,0.3774415969848633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,96,128,1,float16,float16,0,0.6131408214569092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,1,128,1,float16,float16,0,0.15434240102767943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,96,128,1,float16,fp8,0,0.5993872165679932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,1,128,1,float16,fp8,0,0.16490720510482787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,1,128,1,fp8,fp8,0,0.16600799560546875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,96,128,1,fp8,fp8,0,0.6024191856384278
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,2,128,1,float16,float16,0,0.16022399663925171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,2,128,1,float16,fp8,0,0.17236000299453735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,2,128,1,fp8,fp8,0,0.17196639776229858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,4,128,1,float16,float16,0,0.16997439861297609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,4,128,1,float16,fp8,0,0.1797216057777405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,4,128,1,fp8,fp8,0,0.17941440343856813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,8,128,1,float16,float16,0,0.18731679916381835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,8,128,1,float16,fp8,0,0.1989456057548523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,8,128,1,fp8,fp8,0,0.20073599815368653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,96,128,1,float16,float16,0,0.3227871894836426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,96,128,1,float16,fp8,0,0.30918240547180176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,2,128,1,float16,float16,0,0.08978239893913269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,96,128,1,fp8,fp8,0,0.3090303897857666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,1,128,1,float16,float16,0,0.08902400135993957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,1,128,1,float16,fp8,0,0.09683359861373901
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,1,128,1,fp8,fp8,0,0.09636800289154053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,2,128,1,float16,fp8,0,0.09683359861373901
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,2,128,1,fp8,fp8,0,0.09679200053215027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,4,128,1,float16,float16,0,0.09542719721794128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,4,128,1,float16,fp8,0,0.10277279615402221
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,4,128,1,fp8,fp8,0,0.10284960269927979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,8,128,1,float16,float16,0,0.10420479774475097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,96,128,1,fp8,fp8,0,0.16573280096054077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,8,128,1,float16,fp8,0,0.11071360111236572
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,8,128,1,fp8,fp8,0,0.11117119789123535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,96,128,1,float16,float16,0,0.17474080324172975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,96,128,1,float16,fp8,0,0.16418399810791015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,1,128,1,float16,float16,0,0.058222401142120364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,1,128,1,float16,fp8,0,0.057417601346969604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,1,128,1,fp8,fp8,0,0.057868802547454835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,2,128,1,float16,float16,0,0.05838239789009094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,2,128,1,float16,fp8,0,0.058411198854446414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,2,128,1,fp8,fp8,0,0.05822399854660034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,4,128,1,float16,float16,0,0.05944960117340088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,4,128,1,float16,fp8,0,0.05970240235328674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,96,128,1,float16,float16,0,0.09753119945526123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,4,128,1,fp8,fp8,0,0.05954880118370056
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,8,128,1,float16,float16,0,0.06503199934959411
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,8,128,1,float16,fp8,0,0.0663424015045166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,8,128,1,fp8,fp8,0,0.06584479808807372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,96,128,1,float16,fp8,0,0.08943520188331604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,1,128,1,float16,float16,0,0.033425599336624146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,1,128,1,float16,fp8,0,0.03447520136833191
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,96,128,1,fp8,fp8,0,0.08984799981117249
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,1,128,1,fp8,fp8,0,0.03499360084533691
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,2,128,1,float16,float16,0,0.03359679877758026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,2,128,1,float16,fp8,0,0.03456639945507049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,2,128,1,fp8,fp8,0,0.034811198711395264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,4,128,1,float16,float16,0,0.033529600501060484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,4,128,1,float16,fp8,0,0.0352288007736206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,4,128,1,fp8,fp8,0,0.03472639918327332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,8,128,1,float16,float16,0,0.035764798521995544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,8,128,1,float16,fp8,0,0.03586080074310303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,8,128,1,fp8,fp8,0,0.035688000917434695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,96,128,1,float16,float16,0,0.05432800054550171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,96,128,1,float16,fp8,0,0.048502400517463684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,96,128,1,fp8,fp8,0,0.048619198799133304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,1,128,1,float16,float16,0,0.02690559923648834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,1,128,1,float16,fp8,0,0.027710399031639098
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,1,128,1,fp8,fp8,0,0.028004801273345946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,2,128,1,float16,float16,0,0.027195200324058533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,2,128,1,float16,fp8,0,0.028307199478149414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,2,128,1,fp8,fp8,0,0.02805759906768799
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,4,128,1,float16,float16,0,0.027238398790359497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,4,128,1,float16,fp8,0,0.028064000606536865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,4,128,1,fp8,fp8,0,0.028302401304244995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,8,128,1,float16,float16,0,0.027497598528862
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,8,128,1,float16,fp8,0,0.02834399938583374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,8,128,1,fp8,fp8,0,0.02858720123767853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,96,128,1,float16,float16,0,0.03142400085926056
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,96,128,1,float16,fp8,0,0.031516799330711366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,96,128,1,fp8,fp8,0,0.031588798761367796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,1,128,1,float16,float16,0,0.02176959961652756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,1,128,1,float16,fp8,0,0.02244960069656372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,1,128,1,fp8,fp8,0,0.022316800057888032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,2,128,1,float16,float16,0,0.021849599480628968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,2,128,1,float16,fp8,0,0.022780799865722658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,2,128,1,fp8,fp8,0,0.02252320051193237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,4,128,1,float16,float16,0,0.02176000028848648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,4,128,1,float16,fp8,0,0.022870400547981264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,4,128,1,fp8,fp8,0,0.022380800545215608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,8,128,1,float16,float16,0,0.02215680032968521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,8,128,1,float16,fp8,0,0.023155200481414794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,8,128,1,fp8,fp8,0,0.02237280011177063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,1,128,1,float16,float16,0,0.5575200080871582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,1,128,1,float16,fp8,0,0.602675199508667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,1,128,1,fp8,fp8,0,0.6005311965942383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,2,128,1,float16,float16,0,0.5744224071502686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,2,128,1,float16,fp8,0,0.6233263969421386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,2,128,1,fp8,fp8,0,0.6298160076141357
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,4,128,1,float16,float16,0,0.6107120037078857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,4,128,1,float16,fp8,0,0.6614943981170655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,4,128,1,fp8,fp8,0,0.6601200103759766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,8,128,1,float16,float16,0,0.6855631828308105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,8,128,1,float16,fp8,0,0.7409264087677002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,8,128,1,fp8,fp8,0,0.7410255908966065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,1,128,1,float16,float16,0,0.2918704032897949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,1,128,1,float16,fp8,0,0.31397600173950196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,1,128,1,fp8,fp8,0,0.31438560485839845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,2,128,1,float16,float16,0,0.2999840021133423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,96,128,1,float16,float16,0,1.357966423034668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,2,128,1,float16,fp8,0,0.3224384069442749
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,96,128,1,float16,fp8,0,1.3986240386962892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,2,128,1,fp8,fp8,0,0.3215167999267578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,4,128,1,float16,float16,0,0.3177056074142456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,96,128,1,fp8,fp8,0,1.3962207794189454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,4,128,1,float16,fp8,0,0.3406016111373901
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,4,128,1,fp8,fp8,0,0.3436608076095581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,8,128,1,float16,float16,0,0.35238239765167234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,8,128,1,float16,fp8,0,0.38168480396270754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,8,128,1,fp8,fp8,0,0.3811072111129761
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,1,128,1,float16,float16,0,0.15656319856643677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,96,128,1,float16,float16,0,0.6881824016571045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,1,128,1,float16,fp8,0,0.16827199459075928
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,96,128,1,float16,fp8,0,0.7070400238037109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,1,128,1,fp8,fp8,0,0.1683791995048523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,96,128,1,fp8,fp8,0,0.7100751876831055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,2,128,1,float16,float16,0,0.15944160223007203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,2,128,1,float16,fp8,0,0.17505600452423095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,2,128,1,fp8,fp8,0,0.17416000366210938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,4,128,1,float16,float16,0,0.16955039501190186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,4,128,1,float16,fp8,0,0.18042720556259156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,4,128,1,fp8,fp8,0,0.18324480056762696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,8,128,1,float16,float16,0,0.18838560581207275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,8,128,1,float16,fp8,0,0.20371840000152588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,8,128,1,fp8,fp8,0,0.20154879093170167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,96,128,1,float16,float16,0,0.357696008682251
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,96,128,1,float16,fp8,0,0.3634144067764282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,1,128,1,float16,float16,0,0.08923360109329223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,1,128,1,float16,fp8,0,0.09726240038871765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,96,128,1,fp8,fp8,0,0.3663583993911743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,1,128,1,fp8,fp8,0,0.09724159836769104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,4,128,1,float16,fp8,0,0.10393439531326294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,2,128,1,float16,float16,0,0.09015039801597595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,8,128,1,float16,fp8,0,0.11176160573959351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,2,128,1,float16,fp8,0,0.09787520170211791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,2,128,1,fp8,fp8,0,0.09852799773216248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,4,128,1,float16,float16,0,0.09513760209083558
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,4,128,1,fp8,fp8,0,0.103329598903656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,8,128,1,float16,float16,0,0.10432640314102173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,8,128,1,fp8,fp8,0,0.11210559606552124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,96,128,1,float16,float16,0,0.19158719778060912
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,96,128,1,float16,fp8,0,0.1921839952468872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,1,128,1,float16,float16,0,0.0582144021987915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,96,128,1,fp8,fp8,0,0.19179519414901733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,1,128,1,float16,fp8,0,0.057760000228881836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,1,128,1,fp8,fp8,0,0.05737919807434082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,2,128,1,float16,float16,0,0.0580735981464386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,2,128,1,float16,fp8,0,0.05845440030097961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,2,128,1,fp8,fp8,0,0.05785599946975708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,4,128,1,float16,float16,0,0.058905601501464844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,4,128,1,float16,fp8,0,0.05985280275344849
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,4,128,1,fp8,fp8,0,0.05919359922409058
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,8,128,1,float16,float16,0,0.06468799710273743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,8,128,1,float16,fp8,0,0.06636800169944763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,8,128,1,fp8,fp8,0,0.0662671983242035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,96,128,1,float16,float16,0,0.10832959413528442
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,96,128,1,float16,fp8,0,0.10319039821624756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,96,128,1,fp8,fp8,0,0.1033903956413269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,1,128,1,float16,float16,0,0.033137598633766176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,1,128,1,float16,fp8,0,0.03460479974746704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,1,128,1,fp8,fp8,0,0.03484640121459961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,2,128,1,float16,float16,0,0.03317759931087494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,2,128,1,float16,fp8,0,0.03479360044002533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,2,128,1,fp8,fp8,0,0.03476639986038208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,4,128,1,float16,float16,0,0.034238401055336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,4,128,1,float16,fp8,0,0.03508639931678772
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,4,128,1,fp8,fp8,0,0.03474400043487549
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,8,128,1,float16,float16,0,0.03495840132236481
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,8,128,1,float16,fp8,0,0.03571999967098236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,8,128,1,fp8,fp8,0,0.0356687992811203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,96,128,1,float16,float16,0,0.057555198669433594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,96,128,1,float16,fp8,0,0.05229600071907044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,96,128,1,fp8,fp8,0,0.05249760150909424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,1,128,1,float16,float16,0,0.026697599887847902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,1,128,1,float16,fp8,0,0.02817600071430206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,1,128,1,fp8,fp8,0,0.02797439992427826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,2,128,1,float16,float16,0,0.026875200867652892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,2,128,1,fp8,fp8,0,0.028302401304244995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,2,128,1,float16,fp8,0,0.0280239999294281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,4,128,1,float16,float16,0,0.02698720097541809
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,4,128,1,float16,fp8,0,0.028137600421905516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,4,128,1,fp8,fp8,0,0.02810719907283783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,8,128,1,float16,float16,0,0.02715519964694977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,8,128,1,float16,fp8,0,0.028115200996398925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,8,128,1,fp8,fp8,0,0.028420799970626832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,96,128,1,float16,float16,0,0.034088000655174255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,96,128,1,float16,fp8,0,0.03566080033779144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,96,128,1,fp8,fp8,0,0.03550719916820526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,1,128,1,float16,float16,0,0.021953600645065307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,1,128,1,float16,fp8,0,0.02244960069656372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,1,128,1,fp8,fp8,0,0.022606399655342103
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,2,128,1,float16,float16,0,0.021726399660110474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,2,128,1,float16,fp8,0,0.02245119959115982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,2,128,1,fp8,fp8,0,0.02268480062484741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,4,128,1,float16,float16,0,0.021982400119304656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,4,128,1,float16,fp8,0,0.02244639992713928
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,4,128,1,fp8,fp8,0,0.022467200458049775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,8,128,1,float16,float16,0,0.021967999637126923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,8,128,1,float16,fp8,0,0.022804799675941467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,8,128,1,fp8,fp8,0,0.02237440049648285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,96,128,1,float16,float16,0,0.024142399430274963
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,96,128,1,float16,fp8,0,0.02486560046672821
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,96,128,1,fp8,fp8,0,0.025097599625587462
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,1,128,1,float16,float16,0,0.021158400177955627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,1,128,1,float16,fp8,0,0.021729600429534913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,1,128,1,fp8,fp8,0,0.022070400416851044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,2,128,1,float16,float16,0,0.021279999613761903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,2,128,1,float16,fp8,0,0.02196960002183914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,2,128,1,fp8,fp8,0,0.022265599668025972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,4,128,1,float16,float16,0,0.021379199624061585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,4,128,1,float16,fp8,0,0.022203199565410614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,4,128,1,fp8,fp8,0,0.02189600020647049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,8,128,1,float16,float16,0,0.021564799547195434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,8,128,1,float16,fp8,0,0.022099199891090392
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,8,128,1,fp8,fp8,0,0.02222079932689667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,1,128,1,float16,float16,0,0.29192800521850587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,1,128,1,float16,fp8,0,0.31775360107421874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,1,128,1,fp8,fp8,0,0.3207551956176758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,2,128,1,float16,float16,0,0.3008608102798462
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,2,128,1,float16,fp8,0,0.3240943908691406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,2,128,1,fp8,fp8,0,0.326311993598938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,4,128,1,float16,float16,0,0.31916160583496095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,4,128,1,float16,fp8,0,0.3426271915435791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,4,128,1,fp8,fp8,0,0.3450144052505493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,8,128,1,float16,float16,0,0.35539519786834717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,8,128,1,float16,fp8,0,0.38393919467926024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,8,128,1,fp8,fp8,0,0.3818095922470093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,1,128,1,float16,float16,0,0.1554640054702759
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,1,128,1,float16,fp8,0,0.17000000476837157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,96,128,1,float16,float16,0,0.8666687965393066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,1,128,1,fp8,fp8,0,0.17180800437927246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,2,128,1,float16,float16,0,0.16349760293960572
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,96,128,1,float16,fp8,0,0.9070351600646973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,96,128,1,fp8,fp8,0,0.907703971862793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,2,128,1,float16,fp8,0,0.17318079471588135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,2,128,1,fp8,fp8,0,0.1758496046066284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,4,128,1,float16,float16,0,0.1711967945098877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,4,128,1,float16,fp8,0,0.18208800554275512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,4,128,1,fp8,fp8,0,0.18356800079345703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,8,128,1,float16,float16,0,0.19057760238647461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,8,128,1,float16,fp8,0,0.20180160999298097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,8,128,1,fp8,fp8,0,0.2020751953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,96,128,1,float16,float16,0,0.4480576038360596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,1,128,1,float16,float16,0,0.09090240001678467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,96,128,1,float16,fp8,0,0.4689055919647217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,96,128,1,fp8,fp8,0,0.4679999828338623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,1,128,1,float16,fp8,0,0.0984879970550537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,1,128,1,fp8,fp8,0,0.09829440116882324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,2,128,1,float16,float16,0,0.09023839831352234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,2,128,1,float16,fp8,0,0.09934880137443543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,2,128,1,fp8,fp8,0,0.09907360076904297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,4,128,1,float16,float16,0,0.09764959812164306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,4,128,1,float16,fp8,0,0.10448640584945679
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,4,128,1,fp8,fp8,0,0.10486400127410889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,8,128,1,float16,float16,0,0.10673919916152955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,8,128,1,float16,fp8,0,0.11271840333938599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,8,128,1,fp8,fp8,0,0.11289759874343872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,96,128,1,float16,float16,0,0.2349951982498169
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,96,128,1,float16,fp8,0,0.246561598777771
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,96,128,1,fp8,fp8,0,0.2461168050765991
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,1,128,1,float16,float16,0,0.059169602394104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,1,128,1,float16,fp8,0,0.058564800024032596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,1,128,1,fp8,fp8,0,0.058433598279953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,2,128,1,float16,float16,0,0.05899519920349121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,2,128,1,float16,fp8,0,0.05870400071144104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,2,128,1,fp8,fp8,0,0.05896000266075134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,4,128,1,float16,float16,0,0.05957440137863159
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,4,128,1,float16,fp8,0,0.059217602014541626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,96,128,1,float16,fp8,0,0.13148159980773927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,96,128,1,fp8,fp8,0,0.13182879686355592
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,4,128,1,fp8,fp8,0,0.059875202178955075
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,8,128,1,float16,float16,0,0.06537280082702637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,8,128,1,float16,fp8,0,0.06586400270462037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,8,128,1,fp8,fp8,0,0.06622239947319031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,96,128,1,float16,float16,0,0.12933119535446166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,1,128,1,float16,float16,0,0.03290880024433136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,1,128,1,float16,fp8,0,0.03490079939365387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,1,128,1,fp8,fp8,0,0.03524320125579834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,2,128,1,float16,float16,0,0.03371039927005768
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,2,128,1,float16,fp8,0,0.03473759889602661
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,2,128,1,fp8,fp8,0,0.03479839861392975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,4,128,1,float16,float16,0,0.03335840106010437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,4,128,1,float16,fp8,0,0.03489919900894165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,4,128,1,fp8,fp8,0,0.034815999865531924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,8,128,1,float16,float16,0,0.03430399894714355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,8,128,1,float16,fp8,0,0.03604159951210022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,8,128,1,fp8,fp8,0,0.03562560081481934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,96,128,1,float16,float16,0,0.06898880004882812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,96,128,1,float16,fp8,0,0.06569920182228088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,96,128,1,fp8,fp8,0,0.06616160273551941
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,1,128,1,float16,float16,0,0.026825600862503053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,1,128,1,float16,fp8,0,0.028355199098587035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,1,128,1,fp8,fp8,0,0.02828960120677948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,2,128,1,float16,float16,0,0.026907199621200563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,2,128,1,float16,fp8,0,0.028336000442504884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,2,128,1,fp8,fp8,0,0.02815040051937103
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,4,128,1,float16,float16,0,0.026785600185394286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,4,128,1,float16,fp8,0,0.028249600529670717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,4,128,1,fp8,fp8,0,0.02826879918575287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,8,128,1,float16,float16,0,0.027084800601005554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,8,128,1,float16,fp8,0,0.028492799401283263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,8,128,1,fp8,fp8,0,0.028340798616409302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,96,128,1,float16,float16,0,0.03779520094394684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,96,128,1,float16,fp8,0,0.040038400888442995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,96,128,1,fp8,fp8,0,0.039875200390815733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,1,128,1,float16,float16,0,0.02157759964466095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,1,128,1,float16,fp8,0,0.023044799268245698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,1,128,1,fp8,fp8,0,0.022353599965572356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,2,128,1,float16,float16,0,0.021719999611377716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,8,128,1,float16,float16,0,0.021833600103855134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,2,128,1,float16,fp8,0,0.02236959934234619
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,2,128,1,fp8,fp8,0,0.022579200565814972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,4,128,1,float16,float16,0,0.02195200026035309
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,4,128,1,float16,fp8,0,0.022702400386333466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,4,128,1,fp8,fp8,0,0.022672000527381896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,8,128,1,float16,fp8,0,0.022521600127220154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,8,128,1,fp8,fp8,0,0.022702400386333466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,96,128,1,float16,float16,0,0.027739199995994567
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,96,128,1,float16,fp8,0,0.02929919958114624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,96,128,1,fp8,fp8,0,0.02924000024795532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,1,128,1,float16,float16,0,0.021268799901008606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,1,128,1,float16,fp8,0,0.021908800303936004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,1,128,1,fp8,fp8,0,0.02178560048341751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,2,128,1,float16,float16,0,0.021243199706077576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,2,128,1,float16,fp8,0,0.02189279943704605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,2,128,1,fp8,fp8,0,0.022120000422000886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,4,128,1,float16,float16,0,0.02141280025243759
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,4,128,1,float16,fp8,0,0.02234400063753128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,4,128,1,fp8,fp8,0,0.02242400050163269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,8,128,1,float16,float16,0,0.021583999693393707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,8,128,1,float16,fp8,0,0.022254399955272675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,8,128,1,fp8,fp8,0,0.02227199971675873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,96,128,1,float16,float16,0,0.02186720073223114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,96,128,1,float16,fp8,0,0.02255360037088394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,96,128,1,fp8,fp8,0,0.022756800055503845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,1,128,1,float16,float16,0,0.02099040001630783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,1,128,1,float16,fp8,0,0.021718400716781616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,1,128,1,fp8,fp8,0,0.021691200137138367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,2,128,1,float16,float16,0,0.020937600731849672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,2,128,1,float16,fp8,0,0.02163040041923523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,2,128,1,fp8,fp8,0,0.021529600024223328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,4,128,1,float16,float16,0,0.020777599513530733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,4,128,1,float16,fp8,0,0.021844799816608428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,4,128,1,fp8,fp8,0,0.02224159985780716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,8,128,1,float16,float16,0,0.02113119959831238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,8,128,1,float16,fp8,0,0.021862399578094483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,8,128,1,fp8,fp8,0,0.021953600645065307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,1,128,1,float16,float16,0,0.1580175995826721
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,1,128,1,float16,fp8,0,0.17297600507736205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,1,128,1,fp8,fp8,0,0.17254719734191895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,2,128,1,float16,float16,0,0.1643504023551941
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,2,128,1,float16,fp8,0,0.17724959850311278
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,2,128,1,fp8,fp8,0,0.17547520399093627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,4,128,1,float16,float16,0,0.1722640037536621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,4,128,1,float16,fp8,0,0.18634560108184814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,4,128,1,fp8,fp8,0,0.18416960239410402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,8,128,1,float16,float16,0,0.22136800289154052
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,8,128,1,float16,fp8,0,0.23080320358276368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,8,128,1,fp8,fp8,0,0.23058879375457764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,96,128,1,float16,float16,0,0.6275599956512451
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,1,128,1,float16,float16,0,0.09185119867324829
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,1,128,1,float16,fp8,0,0.09914559721946717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,96,128,1,float16,fp8,0,0.6872975826263428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,4,128,1,float16,float16,0,0.09673920273780823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,1,128,1,fp8,fp8,0,0.09959679841995239
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,96,128,1,fp8,fp8,0,0.6870223999023437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,2,128,1,float16,float16,0,0.09236159920692444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,2,128,1,float16,fp8,0,0.10023360252380371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,8,128,1,fp8,fp8,0,0.12831679582595826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,2,128,1,fp8,fp8,0,0.09952319860458374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,4,128,1,float16,fp8,0,0.1052240014076233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,4,128,1,fp8,fp8,0,0.10484319925308228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,8,128,1,float16,float16,0,0.12045279741287232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,8,128,1,float16,fp8,0,0.12860640287399291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,96,128,1,float16,float16,0,0.32553439140319823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,1,128,1,float16,float16,0,0.0593280017375946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,96,128,1,float16,fp8,0,0.35467360019683836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,96,128,1,fp8,fp8,0,0.35381760597229006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,1,128,1,float16,fp8,0,0.058664000034332274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,1,128,1,fp8,fp8,0,0.058555197715759275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,2,128,1,float16,float16,0,0.06012639999389648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,2,128,1,float16,fp8,0,0.05914239883422852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,2,128,1,fp8,fp8,0,0.05969439744949341
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,4,128,1,float16,float16,0,0.06032000184059143
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,4,128,1,float16,fp8,0,0.05978879928588867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,4,128,1,fp8,fp8,0,0.06046720147132874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,8,128,1,float16,float16,0,0.07379680275917053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,8,128,1,float16,fp8,0,0.07532640099525452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,8,128,1,fp8,fp8,0,0.07368159890174866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,96,128,1,float16,float16,0,0.17445600032806396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,96,128,1,float16,fp8,0,0.18845440149307252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,96,128,1,fp8,fp8,0,0.1870911955833435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,1,128,1,float16,float16,0,0.03325600028038025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,1,128,1,float16,fp8,0,0.03527039885520935
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,1,128,1,fp8,fp8,0,0.035097599029541016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,2,128,1,float16,float16,0,0.03318400084972382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,2,128,1,float16,fp8,0,0.03484480082988739
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,2,128,1,fp8,fp8,0,0.03512639999389648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,4,128,1,float16,float16,0,0.03386560082435608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,4,128,1,float16,fp8,0,0.035359999537467955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,4,128,1,fp8,fp8,0,0.03516640067100525
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,8,128,1,float16,float16,0,0.03858399987220764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,8,128,1,float16,fp8,0,0.04005120098590851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,8,128,1,fp8,fp8,0,0.040041598677635196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,96,128,1,float16,float16,0,0.09197919964790344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,96,128,1,float16,fp8,0,0.09285280108451843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,96,128,1,fp8,fp8,0,0.09336479902267455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,1,128,1,float16,float16,0,0.026929599046707154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,1,128,1,float16,fp8,0,0.028228801488876343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,1,128,1,fp8,fp8,0,0.028169599175453187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,4,128,1,float16,fp8,0,0.028143998980522156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,2,128,1,float16,float16,0,0.026888000965118408
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,2,128,1,float16,fp8,0,0.02831999957561493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,2,128,1,fp8,fp8,0,0.028398400545120238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,4,128,1,float16,float16,0,0.027027198672294618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,4,128,1,fp8,fp8,0,0.028281599283218384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,8,128,1,float16,float16,0,0.026817598938941957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,8,128,1,float16,fp8,0,0.02815519869327545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,8,128,1,fp8,fp8,0,0.02823840081691742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,96,128,1,float16,float16,0,0.049374398589134214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,96,128,1,float16,fp8,0,0.054099202156066895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,96,128,1,fp8,fp8,0,0.053844797611236575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,1,128,1,float16,float16,0,0.021729600429534913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,1,128,1,float16,fp8,0,0.02247840017080307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,1,128,1,fp8,fp8,0,0.02269279956817627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,2,128,1,float16,float16,0,0.021585600078105928
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,2,128,1,float16,fp8,0,0.022777600586414336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,2,128,1,fp8,fp8,0,0.02264160066843033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,4,128,1,float16,float16,0,0.021857599914073943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,4,128,1,float16,fp8,0,0.022657600045204163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,4,128,1,fp8,fp8,0,0.02265920042991638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,8,128,1,float16,float16,0,0.02162719964981079
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,8,128,1,float16,fp8,0,0.02263679951429367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,8,128,1,fp8,fp8,0,0.022439999878406523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,96,128,1,float16,float16,0,0.03175520002841949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,96,128,1,float16,fp8,0,0.03382880091667175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,96,128,1,fp8,fp8,0,0.033873599767684934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,1,128,1,float16,float16,0,0.021244800090789794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,1,128,1,float16,fp8,0,0.021835200488567352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,1,128,1,fp8,fp8,0,0.021756799519062044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,2,128,1,float16,float16,0,0.021507200598716737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,2,128,1,float16,fp8,0,0.02187200039625168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,2,128,1,fp8,fp8,0,0.02228959947824478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,4,128,1,float16,float16,0,0.021515199542045595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,4,128,1,float16,fp8,0,0.022307200729846953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,4,128,1,fp8,fp8,0,0.02203039973974228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,8,128,1,float16,float16,0,0.0213359996676445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,8,128,1,float16,fp8,0,0.021859200298786165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,8,128,1,fp8,fp8,0,0.02253440022468567
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,96,128,1,float16,float16,0,0.025248000025749208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,96,128,1,float16,fp8,0,0.02688480019569397
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,96,128,1,fp8,fp8,0,0.026800000667572023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,1,128,1,float16,float16,0,0.02067199945449829
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,1,128,1,float16,fp8,0,0.021660800278186797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,1,128,1,fp8,fp8,0,0.021588799357414246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,2,128,1,float16,float16,0,0.02080000042915344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,2,128,1,float16,fp8,0,0.021982400119304656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,2,128,1,fp8,fp8,0,0.021619200706481934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,4,128,1,float16,float16,0,0.021110400557518005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,4,128,1,float16,fp8,0,0.021996800601482392
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,4,128,1,fp8,fp8,0,0.021716800332069398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,8,128,1,float16,float16,0,0.02133760005235672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,8,128,1,float16,fp8,0,0.021715199947357176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,8,128,1,fp8,fp8,0,0.021531200408935545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,96,128,1,float16,float16,0,0.021529600024223328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,96,128,1,float16,fp8,0,0.022049599885940553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,96,128,1,fp8,fp8,0,0.02220800071954727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,1,128,1,float16,float16,0,0.020623999834060668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,1,128,1,float16,fp8,0,0.021376000344753267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,1,128,1,fp8,fp8,0,0.021382400393486024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,2,128,1,float16,float16,0,0.020763200521469117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,2,128,1,float16,fp8,0,0.021195200085639954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,8,128,1,fp8,fp8,0,0.021753600239753722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,2,128,1,fp8,fp8,0,0.02152319997549057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,4,128,1,float16,float16,0,0.020716799795627593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,4,128,1,float16,fp8,0,0.02157920002937317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,4,128,1,fp8,fp8,0,0.0213359996676445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,8,128,1,float16,float16,0,0.020852799713611602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,8,128,1,float16,fp8,0,0.021323199570178985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,1,128,1,float16,fp8,0,51.80897216796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,1,128,1,fp8,fp8,0,51.80701904296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,2,128,1,fp8,fp8,0,51.7097412109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,2,128,1,float16,fp8,0,52.1908935546875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,2,128,1,float16,float16,0,60.90103759765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,1,128,1,float16,float16,0,62.01614990234375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,4,128,1,float16,fp8,0,51.528631591796874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,4,128,1,float16,float16,0,62.25531005859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,64,128,1,float16,fp8,0,31.00702819824219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,64,128,1,fp8,fp8,0,30.025103759765624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,1,128,1,float16,float16,0,31.437615966796876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,64,128,1,float16,float16,0,35.924075317382815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,4,128,1,fp8,fp8,0,51.9618408203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,8,128,1,float16,fp8,0,52.638031005859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,8,128,1,fp8,fp8,0,52.793695068359376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,8,128,1,float16,float16,0,62.75318603515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,1,128,1,fp8,fp8,0,24.91740264892578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,1,128,1,float16,fp8,0,26.655096435546874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,2,128,1,float16,fp8,0,25.904910278320312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,2,128,1,float16,float16,0,32.21756286621094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,2,128,1,fp8,fp8,0,26.323147583007813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,4,128,1,float16,fp8,0,26.6753662109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,4,128,1,float16,float16,0,31.11280212402344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,4,128,1,fp8,fp8,0,25.513690185546874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,64,128,1,float16,float16,0,17.615577697753906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,64,128,1,float16,fp8,0,14.739158630371094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,8,128,1,float16,fp8,0,26.494970703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,64,128,1,fp8,fp8,0,14.719132995605468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,8,128,1,fp8,fp8,0,25.95943603515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,8,128,1,float16,float16,0,30.823126220703124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,1,128,1,float16,float16,0,15.128001403808593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,1,128,1,float16,fp8,0,13.50738067626953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,1,128,1,fp8,fp8,0,13.619805908203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,2,128,1,float16,fp8,0,13.332583618164062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,2,128,1,fp8,fp8,0,13.38922576904297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,2,128,1,float16,float16,0,16.182049560546876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,4,128,1,float16,fp8,0,12.728797149658202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,4,128,1,float16,float16,0,16.123268127441406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,4,128,1,fp8,fp8,0,12.956166076660157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,64,128,1,float16,fp8,0,7.4322959899902346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,8,128,1,float16,float16,0,15.618513488769532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,64,128,1,float16,float16,0,8.867966461181641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,64,128,1,fp8,fp8,0,7.420148468017578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,1,128,1,float16,fp8,0,6.302888107299805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,8,128,1,float16,fp8,0,14.108917236328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,1,128,1,float16,float16,0,7.448069000244141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,8,128,1,fp8,fp8,0,13.175535583496094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,1,128,1,fp8,fp8,0,6.227587127685547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,2,128,1,float16,fp8,0,6.258833694458008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,2,128,1,fp8,fp8,0,6.291198348999023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,2,128,1,float16,float16,0,7.501372528076172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,4,128,1,float16,fp8,0,6.921018981933594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,4,128,1,fp8,fp8,0,6.329939270019532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,4,128,1,float16,float16,0,7.990790557861328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,8,128,1,float16,float16,0,7.9537811279296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,8,128,1,float16,fp8,0,6.387339019775391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,8,128,1,fp8,fp8,0,6.372686386108398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,1,128,1,float16,fp8,0,29.689974975585937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,2,128,1,float16,fp8,0,29.449691772460938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,1,128,1,fp8,fp8,0,31.166683959960938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,2,128,1,fp8,fp8,0,29.89154968261719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,1,128,1,float16,float16,0,34.3476318359375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,4,128,1,float16,fp8,0,30.156466674804687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,2,128,1,float16,float16,0,35.03394470214844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,4,128,1,float16,float16,0,35.413388061523435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,64,128,1,float16,fp8,0,17.878001403808593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,64,128,1,float16,float16,0,20.415452575683595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,64,128,1,fp8,fp8,0,17.558822631835938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,1,128,1,float16,float16,0,17.73695526123047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,4,128,1,fp8,fp8,0,30.118988037109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,8,128,1,fp8,fp8,0,30.764810180664064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,8,128,1,float16,fp8,0,31.89407958984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,8,128,1,float16,float16,0,35.67771911621094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,1,128,1,fp8,fp8,0,14.494808959960938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,1,128,1,float16,fp8,0,14.933352661132812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,2,128,1,float16,fp8,0,14.468794250488282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,2,128,1,float16,float16,0,17.890470886230467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,2,128,1,fp8,fp8,0,14.655072021484376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,4,128,1,float16,fp8,0,15.350205993652343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,4,128,1,fp8,fp8,0,14.716494750976562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,4,128,1,float16,float16,0,18.082777404785155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,64,128,1,float16,float16,0,10.272144317626953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,64,128,1,float16,fp8,0,9.012462615966797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,8,128,1,float16,fp8,0,14.88992919921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,8,128,1,float16,float16,0,17.7284912109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,8,128,1,fp8,fp8,0,14.9475341796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,1,128,1,float16,fp8,0,7.303103637695313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,64,128,1,fp8,fp8,0,9.052337646484375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,1,128,1,fp8,fp8,0,7.0784912109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,1,128,1,float16,float16,0,8.726592254638671
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,2,128,1,float16,fp8,0,7.524104309082031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,2,128,1,float16,float16,0,9.243299102783203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,2,128,1,fp8,fp8,0,7.162079620361328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,4,128,1,float16,float16,0,9.509814453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,4,128,1,float16,fp8,0,7.4844001770019535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,4,128,1,fp8,fp8,0,8.038407897949218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,64,128,1,float16,float16,0,5.171033477783203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,8,128,1,float16,fp8,0,7.338642883300781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,64,128,1,float16,fp8,0,4.384748840332032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,8,128,1,float16,float16,0,8.989701080322266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,8,128,1,fp8,fp8,0,7.5694129943847654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,64,128,1,fp8,fp8,0,4.468875122070313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,1,128,1,float16,float16,0,4.309527969360351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,1,128,1,fp8,fp8,0,3.541873550415039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,1,128,1,float16,fp8,0,4.003318405151367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,2,128,1,float16,float16,0,4.389793777465821
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,2,128,1,float16,fp8,0,3.7166671752929688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,2,128,1,fp8,fp8,0,3.6613887786865233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,4,128,1,float16,fp8,0,3.734339141845703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,4,128,1,fp8,fp8,0,3.598209762573242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,4,128,1,float16,float16,0,4.291007995605469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,8,128,1,float16,float16,0,4.548315048217773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,8,128,1,float16,fp8,0,3.651337432861328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,8,128,1,fp8,fp8,0,3.6443519592285156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,1,128,1,fp8,fp8,0,20.943391418457033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,1,128,1,float16,fp8,0,21.257884216308593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,2,128,1,fp8,fp8,0,20.37342529296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,1,128,1,float16,float16,0,24.241372680664064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,4,128,1,float16,fp8,0,20.132809448242188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,2,128,1,float16,fp8,0,21.771560668945312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,2,128,1,float16,float16,0,24.44458770751953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,4,128,1,float16,float16,0,24.26838073730469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,64,128,1,float16,fp8,0,13.504302978515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,64,128,1,float16,float16,0,15.398423767089843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,64,128,1,fp8,fp8,0,12.802511596679688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,1,128,1,float16,float16,0,12.36977767944336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,4,128,1,fp8,fp8,0,20.90067901611328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,8,128,1,float16,fp8,0,21.093031311035155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,8,128,1,fp8,fp8,0,21.327825927734374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,8,128,1,float16,float16,0,25.399514770507814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,1,128,1,float16,fp8,0,10.982619476318359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,1,128,1,fp8,fp8,0,10.34742431640625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,2,128,1,float16,fp8,0,10.22723846435547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,2,128,1,float16,float16,0,12.12815170288086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,2,128,1,fp8,fp8,0,10.33469467163086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,4,128,1,float16,fp8,0,10.315287780761718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,4,128,1,float16,float16,0,12.597100830078125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,4,128,1,fp8,fp8,0,10.485737609863282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,64,128,1,float16,float16,0,7.571043395996094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,8,128,1,float16,fp8,0,10.247019195556641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,64,128,1,float16,fp8,0,6.692739105224609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,8,128,1,fp8,fp8,0,10.555140686035156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,64,128,1,fp8,fp8,0,6.573709106445312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,8,128,1,float16,float16,0,13.061141967773438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,1,128,1,float16,float16,0,5.899089431762695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,1,128,1,float16,fp8,0,5.154256057739258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,1,128,1,fp8,fp8,0,5.2415119171142575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,2,128,1,float16,fp8,0,5.1881359100341795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,2,128,1,fp8,fp8,0,5.088446426391601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,2,128,1,float16,float16,0,6.242510223388672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,4,128,1,fp8,fp8,0,5.0706737518310545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,4,128,1,float16,fp8,0,5.778860855102539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,4,128,1,float16,float16,0,6.381907272338867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,64,128,1,float16,float16,0,3.6985103607177736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,64,128,1,float16,fp8,0,3.379163360595703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,8,128,1,float16,fp8,0,5.2005455017089846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,8,128,1,float16,float16,0,6.509044647216797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,8,128,1,fp8,fp8,0,5.269772720336914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,64,128,1,fp8,fp8,0,3.214985656738281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,1,128,1,float16,fp8,0,2.8575231552124025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,1,128,1,float16,float16,0,3.6091567993164064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,1,128,1,fp8,fp8,0,2.6143280029296876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,2,128,1,float16,fp8,0,2.5194992065429687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,2,128,1,float16,float16,0,2.9264272689819335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,2,128,1,fp8,fp8,0,2.616616058349609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,4,128,1,float16,fp8,0,2.539718437194824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,4,128,1,float16,float16,0,2.9563919067382813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,4,128,1,fp8,fp8,0,2.5315759658813475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,8,128,1,float16,float16,0,3.283860778808594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,8,128,1,float16,fp8,0,2.64553279876709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,8,128,1,fp8,fp8,0,2.567967987060547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,1,128,1,float16,fp8,0,27.34207763671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,1,128,1,fp8,fp8,0,26.554354858398437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,2,128,1,float16,fp8,0,26.705838012695313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,4,128,1,float16,fp8,0,27.45281066894531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,2,128,1,fp8,fp8,0,28.138323974609374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,1,128,1,float16,float16,0,31.42613525390625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,2,128,1,float16,float16,0,32.18319091796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,4,128,1,float16,float16,0,31.78759765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,64,128,1,float16,fp8,0,17.56261749267578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,1,128,1,float16,float16,0,15.741867065429688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,64,128,1,fp8,fp8,0,17.48882293701172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,64,128,1,float16,float16,0,20.944158935546874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,4,128,1,fp8,fp8,0,27.689236450195313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,8,128,1,float16,fp8,0,27.54781494140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,8,128,1,fp8,fp8,0,27.403713989257813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,8,128,1,float16,float16,0,32.31725158691406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,1,128,1,float16,fp8,0,13.100201416015626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,1,128,1,fp8,fp8,0,13.095037841796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,2,128,1,float16,fp8,0,14.549462890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,2,128,1,float16,float16,0,16.451646423339845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,2,128,1,fp8,fp8,0,13.886842346191406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,4,128,1,float16,fp8,0,13.880429077148438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,4,128,1,float16,float16,0,16.70637664794922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,4,128,1,fp8,fp8,0,13.300228881835938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,8,128,1,float16,fp8,0,14.107856750488281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,64,128,1,float16,float16,0,10.183360290527343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,64,128,1,float16,fp8,0,8.781918334960938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,8,128,1,float16,float16,0,17.026040649414064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,1,128,1,float16,fp8,0,6.5108482360839846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,64,128,1,fp8,fp8,0,8.713022613525391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,1,128,1,float16,float16,0,7.863495635986328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,8,128,1,fp8,fp8,0,14.434523010253907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,1,128,1,fp8,fp8,0,6.6884819030761715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,2,128,1,float16,float16,0,7.9393470764160154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,2,128,1,float16,fp8,0,6.523873901367187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,2,128,1,fp8,fp8,0,6.566529846191406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,4,128,1,float16,fp8,0,6.609194946289063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,4,128,1,fp8,fp8,0,7.061065673828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,4,128,1,float16,float16,0,8.118036651611328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,64,128,1,float16,fp8,0,4.360168075561523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,64,128,1,float16,float16,0,4.989262390136719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,1,128,1,float16,fp8,0,3.2791374206542967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,64,128,1,fp8,fp8,0,4.364107131958008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,8,128,1,float16,float16,0,8.14620132446289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,1,128,1,float16,float16,0,4.6589103698730465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,8,128,1,float16,fp8,0,6.732403564453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,8,128,1,fp8,fp8,0,6.730076599121094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,1,128,1,fp8,fp8,0,3.2861713409423827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,2,128,1,float16,fp8,0,3.293561553955078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,2,128,1,fp8,fp8,0,3.257160186767578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,2,128,1,float16,float16,0,3.906699371337891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,4,128,1,fp8,fp8,0,3.350337600708008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,4,128,1,float16,fp8,0,3.582948684692383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,4,128,1,float16,float16,0,3.8873294830322265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,8,128,1,float16,float16,0,4.144166564941406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,64,128,1,float16,float16,0,2.389806365966797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,64,128,1,float16,fp8,0,2.2277631759643555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,1,128,1,float16,fp8,0,1.6997007369995116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,1,128,1,float16,float16,0,2.1279199600219725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,64,128,1,fp8,fp8,0,2.202676773071289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,8,128,1,float16,fp8,0,3.4382320404052735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,8,128,1,fp8,fp8,0,3.4244529724121096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,1,128,1,fp8,fp8,0,1.7678815841674804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,2,128,1,float16,fp8,0,1.6596704483032227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,2,128,1,fp8,fp8,0,1.6470607757568358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,2,128,1,float16,float16,0,1.993227195739746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,4,128,1,float16,fp8,0,1.6677568435668946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,4,128,1,float16,float16,0,2.114465522766113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,4,128,1,fp8,fp8,0,1.6652271270751953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,8,128,1,float16,fp8,0,1.6962560653686523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,8,128,1,float16,float16,0,1.8846752166748046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,8,128,1,fp8,fp8,0,1.7533088684082032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,1,128,1,fp8,fp8,0,15.574674987792969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,2,128,1,float16,fp8,0,15.18384246826172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,1,128,1,float16,fp8,0,15.718569946289062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,2,128,1,fp8,fp8,0,15.615383911132813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,4,128,1,float16,fp8,0,15.558125305175782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,1,128,1,float16,float16,0,18.231410217285156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,4,128,1,float16,float16,0,18.54171142578125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,2,128,1,float16,float16,0,19.255091857910156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,64,128,1,float16,fp8,0,10.822624206542969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,1,128,1,float16,float16,0,9.615425872802735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,64,128,1,float16,float16,0,12.91634521484375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,64,128,1,fp8,fp8,0,10.857574462890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,4,128,1,fp8,fp8,0,15.915805053710937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,8,128,1,float16,fp8,0,16.030123901367187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,8,128,1,fp8,fp8,0,16.09681396484375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,8,128,1,float16,float16,0,19.09185791015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,1,128,1,float16,fp8,0,7.7650909423828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,1,128,1,fp8,fp8,0,7.860545349121094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,2,128,1,float16,fp8,0,7.742626953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,2,128,1,fp8,fp8,0,7.857081604003906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,2,128,1,float16,float16,0,9.508822631835937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,4,128,1,float16,fp8,0,7.9385986328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,4,128,1,float16,float16,0,9.509836578369141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,4,128,1,fp8,fp8,0,7.63531494140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,64,128,1,float16,fp8,0,5.44128646850586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,8,128,1,float16,fp8,0,8.348566436767578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,64,128,1,float16,float16,0,6.245572662353515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,64,128,1,fp8,fp8,0,5.436608123779297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,1,128,1,float16,float16,0,4.517300796508789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,8,128,1,float16,float16,0,9.785587310791016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,8,128,1,fp8,fp8,0,8.138030242919921
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,1,128,1,float16,fp8,0,3.742745590209961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,1,128,1,fp8,fp8,0,3.800008010864258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,2,128,1,float16,fp8,0,3.9023681640625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,2,128,1,fp8,fp8,0,3.8652622222900392
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,2,128,1,float16,float16,0,4.844313430786133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,4,128,1,fp8,fp8,0,3.8459152221679687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,4,128,1,float16,fp8,0,4.054091262817383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,4,128,1,float16,float16,0,4.661183929443359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,8,128,1,float16,float16,0,4.478140640258789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,1,128,1,float16,float16,0,2.097172737121582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,1,128,1,float16,fp8,0,1.994361686706543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,64,128,1,float16,float16,0,2.99135684967041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,1,128,1,fp8,fp8,0,2.099900817871094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,64,128,1,fp8,fp8,0,2.731278419494629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,8,128,1,float16,fp8,0,3.96383056640625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,64,128,1,float16,fp8,0,2.8965791702270507
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,8,128,1,fp8,fp8,0,3.969929504394531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,2,128,1,float16,fp8,0,1.9051727294921874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,2,128,1,float16,float16,0,2.308219146728516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,2,128,1,fp8,fp8,0,1.9154720306396484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,4,128,1,float16,fp8,0,1.9512752532958983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,4,128,1,float16,float16,0,2.1619983673095704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,4,128,1,fp8,fp8,0,1.9393280029296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,8,128,1,float16,fp8,0,1.9839759826660157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,8,128,1,float16,float16,0,2.394550323486328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,1,128,1,float16,float16,0,1.1081680297851562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,64,128,1,float16,fp8,0,1.3849984169006349
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,64,128,1,float16,float16,0,1.5765104293823242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,1,128,1,float16,fp8,0,0.9613712310791016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,64,128,1,fp8,fp8,0,1.4017711639404298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,8,128,1,fp8,fp8,0,1.981203269958496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,1,128,1,fp8,fp8,0,1.1758432388305664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,2,128,1,float16,float16,0,1.0519984245300293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,2,128,1,float16,fp8,0,0.966062355041504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,2,128,1,fp8,fp8,0,1.0511024475097657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,4,128,1,float16,float16,0,1.0597200393676758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,4,128,1,float16,fp8,0,0.9803327560424805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,4,128,1,fp8,fp8,0,0.9764927864074707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,8,128,1,float16,float16,0,1.1008399963378905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,8,128,1,fp8,fp8,0,1.004817581176758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,8,128,1,float16,fp8,0,1.0696000099182128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,1,128,1,fp8,fp8,0,14.198980712890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,1,128,1,float16,fp8,0,14.640283203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,2,128,1,fp8,fp8,0,14.284732055664062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,1,128,1,float16,float16,0,16.459483337402343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,2,128,1,float16,fp8,0,16.092575073242188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,2,128,1,float16,float16,0,16.879740905761718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,4,128,1,float16,float16,0,17.424832153320313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,4,128,1,float16,fp8,0,14.543142700195313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,64,128,1,float16,fp8,0,11.619789123535156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,1,128,1,float16,float16,0,8.404801940917968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,4,128,1,fp8,fp8,0,14.929718017578125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,64,128,1,float16,float16,0,13.07816162109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,64,128,1,fp8,fp8,0,11.430574035644531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,8,128,1,float16,fp8,0,15.4135986328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,8,128,1,fp8,fp8,0,15.539814758300782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,8,128,1,float16,float16,0,18.193482971191408
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,1,128,1,fp8,fp8,0,7.080770874023438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,1,128,1,float16,fp8,0,7.418402862548828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,2,128,1,fp8,fp8,0,7.1335090637207035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,2,128,1,float16,fp8,0,7.682790374755859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,2,128,1,float16,float16,0,8.914595031738282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,4,128,1,float16,fp8,0,7.621443176269532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,4,128,1,float16,float16,0,9.42291488647461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,4,128,1,fp8,fp8,0,7.553438568115235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,8,128,1,float16,fp8,0,7.573292541503906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,1,128,1,float16,float16,0,4.136528015136719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,64,128,1,float16,fp8,0,5.879971313476562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,1,128,1,float16,fp8,0,3.551193618774414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,64,128,1,float16,float16,0,6.705411529541015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,8,128,1,fp8,fp8,0,7.518962860107422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,64,128,1,fp8,fp8,0,5.726084899902344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,8,128,1,float16,float16,0,10.278553771972657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,1,128,1,fp8,fp8,0,3.5442176818847657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,2,128,1,float16,fp8,0,3.5756736755371095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,2,128,1,fp8,fp8,0,3.6403472900390623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,2,128,1,float16,float16,0,4.109932708740234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,4,128,1,float16,fp8,0,3.634299087524414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,4,128,1,fp8,fp8,0,3.6122928619384767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,4,128,1,float16,float16,0,4.476103973388672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,8,128,1,float16,float16,0,4.380377578735351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,1,128,1,float16,float16,0,2.004091262817383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,1,128,1,float16,fp8,0,1.9219295501708984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,64,128,1,float16,float16,0,3.217660903930664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,64,128,1,float16,fp8,0,2.929142379760742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,64,128,1,fp8,fp8,0,2.8823471069335938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,8,128,1,float16,fp8,0,3.818590545654297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,8,128,1,fp8,fp8,0,3.7796142578125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,1,128,1,fp8,fp8,0,1.9476095199584962
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,2,128,1,float16,fp8,0,1.8376720428466797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,2,128,1,float16,float16,0,2.0759872436523437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,2,128,1,fp8,fp8,0,1.915559959411621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,4,128,1,float16,fp8,0,1.8347007751464843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,4,128,1,float16,float16,0,2.058907127380371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,4,128,1,fp8,fp8,0,1.836529541015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,8,128,1,float16,float16,0,2.091035270690918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,8,128,1,float16,fp8,0,1.901473617553711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,1,128,1,float16,fp8,0,0.909115219116211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,64,128,1,float16,float16,0,1.6110464096069337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,64,128,1,fp8,fp8,0,1.4666751861572265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,1,128,1,float16,float16,0,1.0430224418640137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,64,128,1,float16,fp8,0,1.471787166595459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,8,128,1,fp8,fp8,0,2.024991989135742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,1,128,1,fp8,fp8,0,1.0073920249938966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,2,128,1,float16,float16,0,0.972708797454834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,2,128,1,float16,fp8,0,0.9137855529785156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,2,128,1,fp8,fp8,0,0.9615504264831543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,4,128,1,float16,float16,0,1.008176040649414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,4,128,1,float16,fp8,0,0.9728879928588867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,4,128,1,fp8,fp8,0,0.9332575798034668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,8,128,1,float16,float16,0,1.0294735908508301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,8,128,1,float16,fp8,0,1.0000495910644531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,64,128,1,float16,float16,0,0.7846240043640137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,8,128,1,fp8,fp8,0,0.9710000038146973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,1,128,1,float16,float16,0,0.49930400848388673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,1,128,1,float16,fp8,0,0.46708478927612307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,64,128,1,float16,fp8,0,0.7621024131774903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,64,128,1,fp8,fp8,0,0.7813648223876953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,1,128,1,fp8,fp8,0,0.46618080139160156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,2,128,1,float16,float16,0,0.5065167903900146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,2,128,1,float16,fp8,0,0.4694608211517334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,2,128,1,fp8,fp8,0,0.48826398849487307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,4,128,1,float16,float16,0,0.5130671977996826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,4,128,1,float16,fp8,0,0.4895167827606201
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,4,128,1,fp8,fp8,0,0.4902224063873291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,8,128,1,float16,float16,0,0.5383999824523926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,8,128,1,float16,fp8,0,0.49349441528320315
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,8,128,1,fp8,fp8,0,0.5076000213623046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,1,128,1,float16,fp8,0,8.430490875244141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,1,128,1,fp8,fp8,0,8.471607971191407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,2,128,1,float16,fp8,0,8.508432006835937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,1,128,1,float16,float16,0,9.812602996826172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,2,128,1,fp8,fp8,0,8.971405029296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,2,128,1,float16,float16,0,9.77025909423828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,4,128,1,float16,float16,0,10.106324768066406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,4,128,1,float16,fp8,0,8.74155502319336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,4,128,1,fp8,fp8,0,8.700726318359376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,64,128,1,float16,fp8,0,7.577191925048828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,64,128,1,float16,float16,0,8.358164978027343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,8,128,1,float16,fp8,0,9.085708618164062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,8,128,1,fp8,fp8,0,9.135107421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,64,128,1,fp8,fp8,0,7.570123291015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,8,128,1,float16,float16,0,10.600953674316406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,1,128,1,float16,float16,0,4.9865680694580075
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,1,128,1,float16,fp8,0,4.204060745239258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,1,128,1,fp8,fp8,0,4.19994888305664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,2,128,1,float16,fp8,0,4.237088012695312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,2,128,1,fp8,fp8,0,4.278116989135742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,2,128,1,float16,float16,0,5.377721786499023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,4,128,1,float16,fp8,0,4.363915252685547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,4,128,1,float16,float16,0,5.102822494506836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,4,128,1,fp8,fp8,0,4.383324813842774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,1,128,1,float16,float16,0,2.362803268432617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,1,128,1,float16,fp8,0,2.1163455963134767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,8,128,1,float16,fp8,0,4.5620849609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,64,128,1,float16,float16,0,4.049801635742187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,8,128,1,float16,float16,0,5.592012786865235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,8,128,1,fp8,fp8,0,4.544100952148438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,64,128,1,fp8,fp8,0,3.799977493286133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,64,128,1,float16,fp8,0,3.9597393035888673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,1,128,1,fp8,fp8,0,2.140278434753418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,2,128,1,float16,float16,0,2.458892822265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,2,128,1,float16,fp8,0,2.125118446350098
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,2,128,1,fp8,fp8,0,2.263230323791504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,4,128,1,float16,fp8,0,2.183336067199707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,4,128,1,float16,float16,0,2.5154607772827147
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,4,128,1,fp8,fp8,0,2.1922847747802736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,8,128,1,float16,float16,0,2.6609615325927733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,8,128,1,float16,fp8,0,2.3464160919189454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,1,128,1,float16,float16,0,1.2180527687072753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,1,128,1,float16,fp8,0,1.073516845703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,64,128,1,float16,float16,0,2.025649642944336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,8,128,1,fp8,fp8,0,2.452191925048828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,64,128,1,float16,fp8,0,2.006884765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,64,128,1,fp8,fp8,0,1.9132303237915038
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,1,128,1,fp8,fp8,0,1.2466815948486327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,2,128,1,float16,float16,0,1.159444808959961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,2,128,1,float16,fp8,0,1.1477935791015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,2,128,1,fp8,fp8,0,1.0853487968444824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,4,128,1,float16,float16,0,1.1969823837280273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,4,128,1,float16,fp8,0,1.1108271598815918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,4,128,1,fp8,fp8,0,1.125767993927002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,8,128,1,float16,float16,0,1.2418208122253418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,8,128,1,float16,fp8,0,1.209603214263916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,64,128,1,float16,float16,0,1.0407648086547852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,1,128,1,float16,float16,0,0.5839263916015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,8,128,1,fp8,fp8,0,1.246072006225586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,64,128,1,fp8,fp8,0,1.0219120025634765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,64,128,1,float16,fp8,0,0.9698464393615722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,1,128,1,float16,fp8,0,0.5513088226318359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,1,128,1,fp8,fp8,0,0.5675759792327881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,2,128,1,float16,float16,0,0.5863999843597412
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,2,128,1,float16,fp8,0,0.5661615848541259
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,2,128,1,fp8,fp8,0,0.5805488109588623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,4,128,1,float16,float16,0,0.6136079788208008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,8,128,1,float16,float16,0,0.6187376022338867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,4,128,1,float16,fp8,0,0.5870192050933838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,4,128,1,fp8,fp8,0,0.5793024063110351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,64,128,1,float16,float16,0,0.5347871780395508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,8,128,1,float16,fp8,0,0.609494400024414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,8,128,1,fp8,fp8,0,0.5951488018035889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,64,128,1,float16,fp8,0,0.5091263771057128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,1,128,1,float16,float16,0,0.3006256103515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,64,128,1,fp8,fp8,0,0.5105264186859131
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,1,128,1,float16,fp8,0,0.29305119514465333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,1,128,1,fp8,fp8,0,0.2941231966018677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,2,128,1,float16,float16,0,0.30787360668182373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,2,128,1,float16,fp8,0,0.2965791940689087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,2,128,1,fp8,fp8,0,0.29379520416259763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,4,128,1,float16,float16,0,0.3194943904876709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,4,128,1,float16,fp8,0,0.2976752042770386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,4,128,1,fp8,fp8,0,0.3020064115524292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,8,128,1,float16,float16,0,0.32543680667877195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,8,128,1,float16,fp8,0,0.31804800033569336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,8,128,1,fp8,fp8,0,0.3161344051361084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,1,128,1,float16,fp8,0,8.449890899658204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,1,128,1,fp8,fp8,0,8.344064331054687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,1,128,1,float16,float16,0,9.228443145751953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,2,128,1,float16,fp8,0,8.416395568847657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,2,128,1,fp8,fp8,0,8.449454498291015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,2,128,1,float16,float16,0,9.444556427001952
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,4,128,1,float16,fp8,0,8.698582458496094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,4,128,1,float16,float16,0,9.754753875732423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,1,128,1,float16,float16,0,4.609028625488281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,4,128,1,fp8,fp8,0,8.840020751953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,8,128,1,float16,fp8,0,9.161262512207031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,64,128,1,float16,float16,0,9.1604736328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,8,128,1,fp8,fp8,0,9.219729614257812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,64,128,1,fp8,fp8,0,8.570275115966798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,64,128,1,float16,fp8,0,8.595751953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,8,128,1,float16,float16,0,10.639997100830078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,1,128,1,float16,fp8,0,4.311870574951172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,1,128,1,fp8,fp8,0,4.125012969970703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,2,128,1,float16,fp8,0,4.2103729248046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,2,128,1,fp8,fp8,0,4.2122753143310545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,2,128,1,float16,float16,0,4.766329574584961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,4,128,1,float16,fp8,0,4.3603263854980465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,4,128,1,float16,float16,0,4.957838439941407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,4,128,1,fp8,fp8,0,4.455648040771484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,1,128,1,float16,float16,0,2.248862457275391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,8,128,1,float16,float16,0,5.147934341430664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,8,128,1,float16,fp8,0,4.615811157226562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,8,128,1,fp8,fp8,0,4.614478302001953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,64,128,1,float16,float16,0,4.599148941040039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,64,128,1,float16,fp8,0,4.312371063232422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,64,128,1,fp8,fp8,0,4.341580963134765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,1,128,1,float16,fp8,0,2.1843183517456053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,1,128,1,fp8,fp8,0,2.16680793762207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,2,128,1,float16,fp8,0,2.1122127532958985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,2,128,1,float16,float16,0,2.2871599197387695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,2,128,1,fp8,fp8,0,2.108616065979004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,4,128,1,float16,fp8,0,2.1925567626953124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,4,128,1,fp8,fp8,0,2.1896623611450194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,4,128,1,float16,float16,0,2.3824640274047852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,8,128,1,float16,fp8,0,2.362857627868652
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,8,128,1,float16,float16,0,2.807851219177246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,1,128,1,float16,float16,0,1.1026639938354492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,1,128,1,float16,fp8,0,1.1819536209106445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,8,128,1,fp8,fp8,0,2.319107246398926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,64,128,1,float16,float16,0,2.303536033630371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,64,128,1,float16,fp8,0,2.159708786010742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,64,128,1,fp8,fp8,0,2.1757791519165037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,1,128,1,fp8,fp8,0,1.0582719802856446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,2,128,1,float16,fp8,0,1.1078399658203124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,2,128,1,float16,float16,0,1.2620863914489746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,2,128,1,fp8,fp8,0,1.1099791526794434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,4,128,1,float16,float16,0,1.1583328247070312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,4,128,1,fp8,fp8,0,1.0985407829284668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,4,128,1,float16,fp8,0,1.142039966583252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,8,128,1,float16,float16,0,1.2491647720336914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,8,128,1,float16,fp8,0,1.1768128395080566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,8,128,1,fp8,fp8,0,1.1922911643981933
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,64,128,1,float16,float16,0,1.2214336395263672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,64,128,1,float16,fp8,0,1.132913589477539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,1,128,1,float16,float16,0,0.560537576675415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,1,128,1,float16,fp8,0,0.5654047966003418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,64,128,1,fp8,fp8,0,1.193175983428955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,1,128,1,fp8,fp8,0,0.5868800163269043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,2,128,1,float16,float16,0,0.5703919887542724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,2,128,1,float16,fp8,0,0.5565616130828858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,2,128,1,fp8,fp8,0,0.568609619140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,4,128,1,float16,float16,0,0.5992752075195312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,4,128,1,float16,fp8,0,0.5646975994110107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,4,128,1,fp8,fp8,0,0.5800000190734863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,8,128,1,float16,float16,0,0.6519616127014161
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,8,128,1,float16,fp8,0,0.6096735954284668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,8,128,1,fp8,fp8,0,0.6096975803375244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,64,128,1,float16,float16,0,0.5884975910186767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,64,128,1,float16,fp8,0,0.5714896202087403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,1,128,1,float16,float16,0,0.2989840030670166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,64,128,1,fp8,fp8,0,0.5769440174102783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,1,128,1,float16,fp8,0,0.29209280014038086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,1,128,1,fp8,fp8,0,0.2912384033203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,2,128,1,float16,float16,0,0.3017263889312744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,2,128,1,float16,fp8,0,0.2933903932571411
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,2,128,1,fp8,fp8,0,0.288974404335022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,8,128,1,float16,fp8,0,0.3187839984893799
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,4,128,1,float16,float16,0,0.3088223934173584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,4,128,1,float16,fp8,0,0.30217440128326417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,4,128,1,fp8,fp8,0,0.30117919445037844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,1,128,1,float16,float16,0,0.16396160125732423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,8,128,1,float16,float16,0,0.3272176027297974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,8,128,1,fp8,fp8,0,0.31331839561462405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,64,128,1,float16,float16,0,0.312940788269043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,64,128,1,float16,fp8,0,0.30139200687408446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,64,128,1,fp8,fp8,0,0.30083041191101073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,1,128,1,float16,fp8,0,0.15968159437179566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,1,128,1,fp8,fp8,0,0.15867999792099
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,2,128,1,float16,float16,0,0.1645151972770691
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,2,128,1,float16,fp8,0,0.1556864023208618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,2,128,1,fp8,fp8,0,0.1603216052055359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,4,128,1,float16,float16,0,0.16896159648895265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,4,128,1,float16,fp8,0,0.16583679914474486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,4,128,1,fp8,fp8,0,0.1644719958305359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,8,128,1,float16,float16,0,0.17997759580612183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,8,128,1,float16,fp8,0,0.17283999919891357
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,8,128,1,fp8,fp8,0,0.1731503963470459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,1,128,1,float16,fp8,0,5.148518371582031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,1,128,1,float16,float16,0,5.549372863769531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,1,128,1,fp8,fp8,0,5.184974288940429
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,2,128,1,float16,fp8,0,5.272339248657227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,2,128,1,fp8,fp8,0,5.230350494384766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,2,128,1,float16,float16,0,5.709187316894531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,4,128,1,float16,fp8,0,5.474185562133789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,4,128,1,float16,float16,0,6.061164855957031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,4,128,1,fp8,fp8,0,5.448910522460937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,8,128,1,float16,fp8,0,5.836902236938476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,8,128,1,float16,float16,0,6.325028610229492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,8,128,1,fp8,fp8,0,5.861260986328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,64,128,1,float16,fp8,0,5.946860885620117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,64,128,1,float16,float16,0,6.231884765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,64,128,1,fp8,fp8,0,5.978163146972657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,1,128,1,float16,float16,0,2.820635223388672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,1,128,1,float16,fp8,0,2.56854248046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,1,128,1,fp8,fp8,0,2.568998336791992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,2,128,1,float16,float16,0,2.722908782958984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,2,128,1,float16,fp8,0,2.6166559219360352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,2,128,1,fp8,fp8,0,2.679377555847168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,4,128,1,float16,float16,0,2.9496320724487304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,4,128,1,float16,fp8,0,2.77148323059082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,4,128,1,fp8,fp8,0,2.721486473083496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,8,128,1,float16,float16,0,3.2109214782714846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,8,128,1,fp8,fp8,0,2.926260757446289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,8,128,1,float16,fp8,0,3.09036808013916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,1,128,1,float16,float16,0,1.3529696464538574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,64,128,1,float16,float16,0,3.0980031967163084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,1,128,1,float16,fp8,0,1.2969552040100099
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,64,128,1,float16,fp8,0,3.0135536193847656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,64,128,1,fp8,fp8,0,3.135286331176758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,1,128,1,fp8,fp8,0,1.3764351844787597
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,2,128,1,float16,fp8,0,1.3272175788879395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,2,128,1,float16,float16,0,1.384932804107666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,2,128,1,fp8,fp8,0,1.431984043121338
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,4,128,1,float16,float16,0,1.443075180053711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,4,128,1,float16,fp8,0,1.372321605682373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,4,128,1,fp8,fp8,0,1.3809215545654296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,8,128,1,float16,float16,0,1.5425423622131347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,8,128,1,float16,fp8,0,1.491153621673584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,1,128,1,float16,float16,0,0.6785888195037841
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,1,128,1,float16,fp8,0,0.6633376121520996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,8,128,1,fp8,fp8,0,1.4821311950683593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,64,128,1,float16,fp8,0,1.5448368072509766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,64,128,1,float16,float16,0,1.5822319984436035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,1,128,1,fp8,fp8,0,0.7013487815856934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,64,128,1,fp8,fp8,0,1.5739071846008301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,2,128,1,float16,fp8,0,0.6810783863067627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,2,128,1,fp8,fp8,0,0.6762576103210449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,2,128,1,float16,float16,0,0.6931424140930176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,4,128,1,float16,float16,0,0.7235455989837647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,4,128,1,float16,fp8,0,0.7059775829315186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,4,128,1,fp8,fp8,0,0.7097599983215332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,8,128,1,float16,float16,0,0.8178591728210449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,8,128,1,float16,fp8,0,0.7666575908660889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,8,128,1,fp8,fp8,0,0.7592063903808594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,64,128,1,float16,float16,0,0.7942671775817871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,64,128,1,float16,fp8,0,0.7744607925415039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,1,128,1,float16,float16,0,0.3558687925338745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,1,128,1,float16,fp8,0,0.3469280004501343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,64,128,1,fp8,fp8,0,0.7685855865478516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,1,128,1,fp8,fp8,0,0.353222393989563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,2,128,1,float16,float16,0,0.3654367923736572
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,2,128,1,float16,fp8,0,0.3541968107223511
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,2,128,1,fp8,fp8,0,0.35273280143737795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,4,128,1,float16,float16,0,0.38069279193878175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,4,128,1,float16,fp8,0,0.36253440380096436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,4,128,1,fp8,fp8,0,0.3678800106048584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,8,128,1,float16,float16,0,0.4110447883605957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,8,128,1,float16,fp8,0,0.39687678813934324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,8,128,1,fp8,fp8,0,0.3964047908782959
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,64,128,1,float16,float16,0,0.4136415958404541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,64,128,1,float16,fp8,0,0.4041776180267334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,1,128,1,float16,float16,0,0.19110560417175293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,64,128,1,fp8,fp8,0,0.40705599784851076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,1,128,1,float16,fp8,0,0.18896960020065307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,1,128,1,fp8,fp8,0,0.1892688035964966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,4,128,1,fp8,fp8,0,0.19651999473571777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,2,128,1,float16,float16,0,0.1944416046142578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,2,128,1,float16,fp8,0,0.1898095965385437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,2,128,1,fp8,fp8,0,0.1872383952140808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,4,128,1,float16,float16,0,0.20031681060791015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,4,128,1,float16,fp8,0,0.1932047963142395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,8,128,1,float16,float16,0,0.21584479808807372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,8,128,1,float16,fp8,0,0.2109503984451294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,8,128,1,fp8,fp8,0,0.21045119762420655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,64,128,1,float16,float16,0,0.22306399345397948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,64,128,1,float16,fp8,0,0.21615679264068605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,64,128,1,fp8,fp8,0,0.21235520839691163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,1,128,1,float16,float16,0,0.10942720174789429
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,1,128,1,float16,fp8,0,0.10317120552062989
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,4,128,1,float16,fp8,0,0.10704959630966186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,1,128,1,fp8,fp8,0,0.107368004322052
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,2,128,1,float16,float16,0,0.11096160411834717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,2,128,1,float16,fp8,0,0.10719679594039917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,8,128,1,fp8,fp8,0,0.11562399864196778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,2,128,1,fp8,fp8,0,0.1069551944732666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,4,128,1,float16,float16,0,0.11170879602432252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,4,128,1,fp8,fp8,0,0.10977920293807983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,8,128,1,float16,float16,0,0.12053600549697877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,8,128,1,float16,fp8,0,0.11510399580001832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,1,128,1,float16,float16,0,5.715948867797851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,1,128,1,float16,fp8,0,5.397422409057617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,1,128,1,fp8,fp8,0,5.394996643066406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,2,128,1,float16,float16,0,5.832220840454101
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,2,128,1,float16,fp8,0,5.539868927001953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,2,128,1,fp8,fp8,0,5.537160110473633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,4,128,1,float16,fp8,0,5.835516738891601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,4,128,1,float16,float16,0,6.269588851928711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,4,128,1,fp8,fp8,0,5.79248161315918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,8,128,1,float16,fp8,0,6.313040161132813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,8,128,1,float16,float16,0,6.7275245666503904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,8,128,1,fp8,fp8,0,6.3244270324707035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,64,128,1,float16,float16,0,7.565064239501953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,64,128,1,float16,fp8,0,7.179574584960937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,1,128,1,float16,float16,0,2.8210336685180666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,64,128,1,fp8,fp8,0,7.206838226318359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,1,128,1,float16,fp8,0,2.694175910949707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,1,128,1,fp8,fp8,0,2.8069999694824217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,2,128,1,float16,float16,0,2.9267696380615233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,2,128,1,float16,fp8,0,2.7965280532836916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,2,128,1,fp8,fp8,0,2.7852384567260744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,4,128,1,float16,float16,0,3.056158447265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,4,128,1,float16,fp8,0,2.906163215637207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,4,128,1,fp8,fp8,0,2.9149728775024415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,8,128,1,float16,float16,0,3.3972446441650392
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,8,128,1,float16,fp8,0,3.1885984420776365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,8,128,1,fp8,fp8,0,3.3052352905273437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,1,128,1,float16,float16,0,1.3912591934204102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,1,128,1,float16,fp8,0,1.3665727615356444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,1,128,1,fp8,fp8,0,1.3685152053833007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,2,128,1,float16,float16,0,1.4215344429016112
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,2,128,1,float16,fp8,0,1.4017279624938965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,2,128,1,fp8,fp8,0,1.401863956451416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,64,128,1,float16,fp8,0,3.6191951751708986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,64,128,1,float16,float16,0,3.7982318878173826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,4,128,1,float16,float16,0,1.517084789276123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,64,128,1,fp8,fp8,0,3.6543216705322266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,4,128,1,float16,fp8,0,1.4693167686462403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,4,128,1,fp8,fp8,0,1.4772768020629883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,8,128,1,float16,float16,0,1.65545597076416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,8,128,1,float16,fp8,0,1.6439184188842773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,8,128,1,fp8,fp8,0,1.6110864639282227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,1,128,1,float16,float16,0,0.7114607810974121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,1,128,1,float16,fp8,0,0.6879248142242431
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,64,128,1,float16,float16,0,1.9113103866577148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,64,128,1,float16,fp8,0,1.8368719100952149
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,64,128,1,fp8,fp8,0,1.8318511962890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,1,128,1,fp8,fp8,0,0.7005328178405762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,2,128,1,float16,float16,0,0.741918420791626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,2,128,1,float16,fp8,0,0.7119664192199707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,2,128,1,fp8,fp8,0,0.7217184066772461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,4,128,1,float16,float16,0,0.7573232173919677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,4,128,1,float16,fp8,0,0.7562575817108155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,4,128,1,fp8,fp8,0,0.7605679988861084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,8,128,1,float16,float16,0,0.8420432090759278
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,8,128,1,float16,fp8,0,0.830998420715332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,8,128,1,fp8,fp8,0,0.8327535629272461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,64,128,1,float16,float16,0,0.9511407852172852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,1,128,1,float16,float16,0,0.373473596572876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,1,128,1,float16,fp8,0,0.3675087928771973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,64,128,1,float16,fp8,0,0.943182373046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,64,128,1,fp8,fp8,0,0.9149264335632324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,1,128,1,fp8,fp8,0,0.3615823984146118
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,2,128,1,float16,float16,0,0.38218719959259034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,2,128,1,float16,fp8,0,0.3710223913192749
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,2,128,1,fp8,fp8,0,0.36793758869171145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,4,128,1,float16,float16,0,0.40089759826660154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,4,128,1,float16,fp8,0,0.39269280433654785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,4,128,1,fp8,fp8,0,0.38676159381866454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,8,128,1,float16,float16,0,0.42786240577697754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,8,128,1,float16,fp8,0,0.420030403137207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,1,128,1,float16,fp8,0,0.19270880222320558
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,8,128,1,fp8,fp8,0,0.42125759124755857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,64,128,1,float16,float16,0,0.4928127765655518
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,64,128,1,float16,fp8,0,0.47429118156433103
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,1,128,1,float16,float16,0,0.19956320524215698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,64,128,1,fp8,fp8,0,0.4829360008239746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,1,128,1,fp8,fp8,0,0.18944319486618041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,2,128,1,float16,float16,0,0.20030879974365234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,2,128,1,float16,fp8,0,0.1981328010559082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,2,128,1,fp8,fp8,0,0.19828959703445434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,4,128,1,float16,float16,0,0.20921919345855713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,4,128,1,float16,fp8,0,0.2043247938156128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,4,128,1,fp8,fp8,0,0.20863680839538573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,8,128,1,float16,float16,0,0.23161120414733888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,64,128,1,fp8,fp8,0,0.25571680068969727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,8,128,1,float16,fp8,0,0.2284991979598999
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,8,128,1,fp8,fp8,0,0.22241439819335937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,64,128,1,float16,float16,0,0.260862398147583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,64,128,1,float16,fp8,0,0.25105440616607666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,1,128,1,float16,float16,0,0.10971839427947998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,1,128,1,float16,fp8,0,0.1113711953163147
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,1,128,1,fp8,fp8,0,0.11034879684448243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,2,128,1,float16,float16,0,0.11345759630203248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,2,128,1,float16,fp8,0,0.11185920238494873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,2,128,1,fp8,fp8,0,0.11100640296936035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,4,128,1,float16,float16,0,0.1172063946723938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,4,128,1,float16,fp8,0,0.11579519510269165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,4,128,1,fp8,fp8,0,0.11752159595489502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,8,128,1,float16,float16,0,0.1281216025352478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,8,128,1,float16,fp8,0,0.12472319602966309
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,8,128,1,fp8,fp8,0,0.12475680112838745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,64,128,1,float16,float16,0,0.14424480199813844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,64,128,1,float16,fp8,0,0.13801920413970947
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,64,128,1,fp8,fp8,0,0.13649280071258546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,1,128,1,float16,float16,0,0.06175839900970459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,1,128,1,float16,fp8,0,0.0608959972858429
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,1,128,1,fp8,fp8,0,0.06113759875297546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,2,128,1,float16,float16,0,0.06480799913406372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,2,128,1,float16,fp8,0,0.06074879765510559
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,2,128,1,fp8,fp8,0,0.061534398794174196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,4,128,1,float16,float16,0,0.06615679860115051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,4,128,1,float16,fp8,0,0.06186559796333313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,4,128,1,fp8,fp8,0,0.0627615988254547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,8,128,1,float16,float16,0,0.07552160024642944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,8,128,1,float16,fp8,0,0.06759999990463257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,8,128,1,fp8,fp8,0,0.06837120056152343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,1,128,1,float16,float16,0,3.991379165649414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,1,128,1,float16,fp8,0,3.980908966064453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,1,128,1,fp8,fp8,0,3.9749359130859374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,2,128,1,float16,float16,0,4.129628753662109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,2,128,1,float16,fp8,0,4.127735900878906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,2,128,1,fp8,fp8,0,4.125352096557617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,4,128,1,float16,float16,0,4.476540756225586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,4,128,1,float16,fp8,0,4.381827163696289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,4,128,1,fp8,fp8,0,4.373564910888672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,8,128,1,float16,float16,0,5.011553573608398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,8,128,1,float16,fp8,0,4.948600006103516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,8,128,1,fp8,fp8,0,4.944222259521484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,1,128,1,float16,float16,0,1.989187240600586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,64,128,1,float16,float16,0,6.728148651123047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,1,128,1,float16,fp8,0,1.9791919708251953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,64,128,1,float16,fp8,0,6.545222473144531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,1,128,1,fp8,fp8,0,1.9796367645263673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,2,128,1,float16,float16,0,2.0624176025390626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,2,128,1,float16,fp8,0,2.059841537475586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,2,128,1,fp8,fp8,0,2.0674800872802734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,4,128,1,float16,float16,0,2.2084672927856444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,4,128,1,float16,fp8,0,2.209270477294922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,4,128,1,fp8,fp8,0,2.1855920791625976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,8,128,1,float16,float16,0,2.5241504669189454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,64,128,1,fp8,fp8,0,6.51949462890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,1,128,1,float16,float16,0,1.0020400047302247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,8,128,1,float16,fp8,0,2.484502410888672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,8,128,1,fp8,fp8,0,2.509321594238281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,1,128,1,float16,fp8,0,1.008561611175537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,1,128,1,fp8,fp8,0,1.0101759910583497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,2,128,1,float16,float16,0,1.038755226135254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,2,128,1,float16,fp8,0,1.0386431694030762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,2,128,1,fp8,fp8,0,1.044649600982666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,64,128,1,float16,float16,0,3.3381103515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,64,128,1,float16,fp8,0,3.250193786621094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,4,128,1,float16,float16,0,1.1182559967041015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,64,128,1,fp8,fp8,0,3.275603103637695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,4,128,1,float16,fp8,0,1.1129983901977538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,4,128,1,fp8,fp8,0,1.1027423858642578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,8,128,1,float16,float16,0,1.2733167648315429
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,8,128,1,float16,fp8,0,1.260547161102295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,1,128,1,float16,float16,0,0.5131135940551758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,1,128,1,float16,fp8,0,0.5113791942596435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,8,128,1,fp8,fp8,0,1.2572959899902343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,1,128,1,fp8,fp8,0,0.5151103973388672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,64,128,1,float16,float16,0,1.6765727996826172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,2,128,1,float16,float16,0,0.5348879814147949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,64,128,1,float16,fp8,0,1.6410127639770509
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,2,128,1,float16,fp8,0,0.5298687934875488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,2,128,1,fp8,fp8,0,0.5287631988525391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,64,128,1,fp8,fp8,0,1.6506288528442383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,4,128,1,float16,float16,0,0.5712800025939941
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,4,128,1,float16,fp8,0,0.5765535831451416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,4,128,1,fp8,fp8,0,0.5792880058288574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,8,128,1,float16,float16,0,0.6464543819427491
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,8,128,1,float16,fp8,0,0.6361040115356446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,8,128,1,fp8,fp8,0,0.6343808174133301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,64,128,1,float16,float16,0,0.8502559661865234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,1,128,1,float16,float16,0,0.27519519329071046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,64,128,1,float16,fp8,0,0.8379776000976562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,1,128,1,float16,fp8,0,0.2759936094284058
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,64,128,1,fp8,fp8,0,0.8370528221130371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,1,128,1,fp8,fp8,0,0.26966080665588377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,2,128,1,float16,float16,0,0.2774399995803833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,2,128,1,float16,fp8,0,0.27556960582733153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,8,128,1,float16,float16,0,0.3359663963317871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,2,128,1,fp8,fp8,0,0.27602720260620117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,4,128,1,float16,float16,0,0.30324320793151854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,4,128,1,float16,fp8,0,0.2959775924682617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,4,128,1,fp8,fp8,0,0.30174400806427004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,8,128,1,float16,fp8,0,0.33080480098724363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,8,128,1,fp8,fp8,0,0.3294543981552124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,64,128,1,float16,float16,0,0.4379312038421631
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,64,128,1,float16,fp8,0,0.43155522346496583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,1,128,1,float16,float16,0,0.148579204082489
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,64,128,1,fp8,fp8,0,0.42706880569458006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,4,128,1,float16,float16,0,0.15961600542068483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,1,128,1,float16,fp8,0,0.14658080339431762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,1,128,1,fp8,fp8,0,0.14727360010147095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,2,128,1,float16,float16,0,0.150929594039917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,2,128,1,float16,fp8,0,0.15396480560302733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,2,128,1,fp8,fp8,0,0.15028320550918578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,4,128,1,float16,fp8,0,0.1620336055755615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,4,128,1,fp8,fp8,0,0.16287200450897216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,8,128,1,float16,float16,0,0.1791775941848755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,8,128,1,float16,fp8,0,0.17869280576705932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,8,128,1,fp8,fp8,0,0.17855360507965087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,64,128,1,float16,float16,0,0.2332848072052002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,64,128,1,float16,fp8,0,0.23220961093902587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,64,128,1,fp8,fp8,0,0.2277071952819824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,1,128,1,float16,float16,0,0.08721920251846313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,1,128,1,float16,fp8,0,0.08713120222091675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,1,128,1,fp8,fp8,0,0.08663039803504943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,2,128,1,float16,float16,0,0.08691679835319518
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,2,128,1,float16,fp8,0,0.08698239922523499
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,2,128,1,fp8,fp8,0,0.08801440000534058
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,4,128,1,float16,float16,0,0.09184319972991943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,4,128,1,float16,fp8,0,0.0938543975353241
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,4,128,1,fp8,fp8,0,0.09321920275688171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,8,128,1,float16,float16,0,0.10160319805145264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,8,128,1,float16,fp8,0,0.1005519986152649
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,8,128,1,fp8,fp8,0,0.10060319900512696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,64,128,1,float16,float16,0,0.1317296028137207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,64,128,1,float16,fp8,0,0.12699999809265136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,64,128,1,fp8,fp8,0,0.12873760461807252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,1,128,1,float16,float16,0,0.05135359764099121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,1,128,1,float16,fp8,0,0.04885280132293701
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,1,128,1,fp8,fp8,0,0.048651200532913205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,2,128,1,float16,float16,0,0.05170239806175232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,2,128,1,float16,fp8,0,0.049185600876808164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,2,128,1,fp8,fp8,0,0.048604801297187805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,4,128,1,float16,float16,0,0.05398240089416504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,4,128,1,float16,fp8,0,0.05125920176506042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,4,128,1,fp8,fp8,0,0.05122079849243164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,8,128,1,float16,float16,0,0.062168002128601074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,8,128,1,float16,fp8,0,0.05650079846382141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,8,128,1,fp8,fp8,0,0.05629119873046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,64,128,1,float16,float16,0,0.07593920230865478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,64,128,1,float16,fp8,0,0.06903039813041686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,64,128,1,fp8,fp8,0,0.06868799924850463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,1,128,1,float16,float16,0,0.033024001121521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,1,128,1,float16,fp8,0,0.03495840132236481
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,1,128,1,fp8,fp8,0,0.034755200147628784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,2,128,1,float16,float16,0,0.033211201429367065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,2,128,1,float16,fp8,0,0.03472639918327332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,2,128,1,fp8,fp8,0,0.03461920022964478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,4,128,1,float16,float16,0,0.03383840024471283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,4,128,1,float16,fp8,0,0.034974399209022525
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,4,128,1,fp8,fp8,0,0.035464000701904294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,8,128,1,float16,float16,0,0.03485440015792847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,8,128,1,float16,fp8,0,0.035769599676132205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,8,128,1,fp8,fp8,0,0.03555200099945068
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,1,128,1,float16,float16,0,1.6048704147338868
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,1,128,1,float16,fp8,0,1.6726287841796874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,1,128,1,fp8,fp8,0,1.6633312225341796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,2,128,1,float16,float16,0,1.6783695220947266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,2,128,1,float16,fp8,0,1.7310800552368164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,2,128,1,fp8,fp8,0,1.7330560684204102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,4,128,1,float16,float16,0,1.8245599746704102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,4,128,1,float16,fp8,0,1.8635072708129883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,4,128,1,fp8,fp8,0,1.8723535537719727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,8,128,1,float16,float16,0,2.138484764099121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,8,128,1,float16,fp8,0,2.163369560241699
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,8,128,1,fp8,fp8,0,2.1444320678710938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,1,128,1,float16,float16,0,0.8100031852722168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,1,128,1,float16,fp8,0,0.8432736396789551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,1,128,1,fp8,fp8,0,0.8369551658630371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,2,128,1,float16,float16,0,0.8443903923034668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,2,128,1,float16,fp8,0,0.8704336166381836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,2,128,1,fp8,fp8,0,0.8708016395568847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,4,128,1,float16,float16,0,0.9213392257690429
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,64,128,1,float16,float16,0,3.2334209442138673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,64,128,1,float16,fp8,0,3.079542350769043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,64,128,1,fp8,fp8,0,3.092359924316406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,4,128,1,float16,fp8,0,0.9380751609802246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,4,128,1,fp8,fp8,0,0.9391103744506836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,8,128,1,float16,float16,0,1.0737695693969727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,8,128,1,float16,fp8,0,1.0803680419921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,8,128,1,fp8,fp8,0,1.0791024208068847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,1,128,1,float16,float16,0,0.413259220123291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,1,128,1,float16,fp8,0,0.4306640148162842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,1,128,1,fp8,fp8,0,0.4303408145904541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,2,128,1,float16,float16,0,0.43424158096313475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,2,128,1,float16,fp8,0,0.4467311859130859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,64,128,1,float16,fp8,0,1.5517423629760743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,64,128,1,float16,float16,0,1.620302391052246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,2,128,1,fp8,fp8,0,0.4422272205352783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,64,128,1,fp8,fp8,0,1.5568703651428222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,4,128,1,float16,float16,0,0.47154879570007324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,4,128,1,float16,fp8,0,0.4805488109588623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,4,128,1,fp8,fp8,0,0.48059520721435545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,8,128,1,float16,float16,0,0.548142385482788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,8,128,1,float16,fp8,0,0.5526463985443115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,8,128,1,fp8,fp8,0,0.5521791934967041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,64,128,1,float16,float16,0,0.8155823707580566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,1,128,1,float16,float16,0,0.22079999446868898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,1,128,1,float16,fp8,0,0.22667040824890136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,64,128,1,float16,fp8,0,0.7915296077728271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,1,128,1,fp8,fp8,0,0.2267375946044922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,64,128,1,fp8,fp8,0,0.7796944141387939
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,2,128,1,float16,float16,0,0.22951838970184327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,2,128,1,float16,fp8,0,0.23505280017852784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,2,128,1,fp8,fp8,0,0.23854079246520996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,4,128,1,float16,float16,0,0.24745759963989258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,4,128,1,float16,fp8,0,0.25083041191101074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,4,128,1,fp8,fp8,0,0.25365118980407714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,8,128,1,float16,float16,0,0.2845168113708496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,8,128,1,float16,fp8,0,0.2858128070831299
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,8,128,1,fp8,fp8,0,0.2875391960144043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,64,128,1,float16,float16,0,0.4254928112030029
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,64,128,1,float16,fp8,0,0.4128704071044922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,1,128,1,float16,float16,0,0.12098400592803955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,64,128,1,fp8,fp8,0,0.40425920486450195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,1,128,1,float16,fp8,0,0.12482719421386719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,1,128,1,fp8,fp8,0,0.12427999973297119
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,2,128,1,float16,float16,0,0.12732959985733033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,2,128,1,float16,fp8,0,0.13030879497528075
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,2,128,1,fp8,fp8,0,0.13191360235214233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,4,128,1,float16,float16,0,0.13497600555419922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,4,128,1,float16,fp8,0,0.13761279582977295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,4,128,1,fp8,fp8,0,0.1379439949989319
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,8,128,1,float16,float16,0,0.1542240023612976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,8,128,1,float16,fp8,0,0.1557103991508484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,8,128,1,fp8,fp8,0,0.15552799701690673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,64,128,1,float16,float16,0,0.22714560031890868
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,64,128,1,float16,fp8,0,0.21409120559692382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,64,128,1,fp8,fp8,0,0.21559360027313232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,1,128,1,float16,float16,0,0.07421600222587585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,1,128,1,float16,fp8,0,0.07521119713783264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,1,128,1,fp8,fp8,0,0.07550399899482726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,2,128,1,float16,float16,0,0.07500479817390442
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,2,128,1,float16,fp8,0,0.07620959877967834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,2,128,1,fp8,fp8,0,0.07591840028762817
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,4,128,1,float16,float16,0,0.0807807981967926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,4,128,1,float16,fp8,0,0.08179519772529602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,4,128,1,fp8,fp8,0,0.08167840242385864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,8,128,1,float16,float16,0,0.08885440230369568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,8,128,1,float16,fp8,0,0.08925759792327881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,8,128,1,fp8,fp8,0,0.08952640295028687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,64,128,1,float16,float16,0,0.12549279928207396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,64,128,1,float16,fp8,0,0.11923520565032959
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,64,128,1,fp8,fp8,0,0.11961120367050171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,1,128,1,float16,float16,0,0.04511519968509674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,1,128,1,float16,fp8,0,0.043977600336074826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,1,128,1,fp8,fp8,0,0.04488799870014191
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,2,128,1,float16,float16,0,0.04569920003414154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,2,128,1,float16,fp8,0,0.04383040070533752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,2,128,1,fp8,fp8,0,0.04449760019779205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,4,128,1,float16,float16,0,0.04742720127105713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,4,128,1,float16,fp8,0,0.044964799284935
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,4,128,1,fp8,fp8,0,0.044950398802757266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,8,128,1,float16,float16,0,0.05570399761199951
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,8,128,1,float16,fp8,0,0.05190879702568054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,8,128,1,fp8,fp8,0,0.05108000040054321
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,64,128,1,float16,float16,0,0.07224479913711548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,64,128,1,float16,fp8,0,0.06312000155448913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,64,128,1,fp8,fp8,0,0.0625711977481842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,1,128,1,float16,float16,0,0.030049601197242738
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,1,128,1,float16,fp8,0,0.031044799089431762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,1,128,1,fp8,fp8,0,0.031431999802589414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,2,128,1,float16,float16,0,0.029960000514984132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,2,128,1,float16,fp8,0,0.03129119873046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,2,128,1,fp8,fp8,0,0.031222400069236756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,4,128,1,float16,float16,0,0.03065760135650635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,4,128,1,float16,fp8,0,0.031411200761795044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,4,128,1,fp8,fp8,0,0.031615999341011045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,8,128,1,float16,float16,0,0.03139680027961731
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,8,128,1,float16,fp8,0,0.03243840038776398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,8,128,1,fp8,fp8,0,0.032583999633789065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,64,128,1,float16,float16,0,0.03790239989757538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,64,128,1,float16,fp8,0,0.037545600533485414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,64,128,1,fp8,fp8,0,0.03753759860992432
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,1,128,1,float16,float16,0,0.023996800184249878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,1,128,1,float16,fp8,0,0.024884800612926482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,1,128,1,fp8,fp8,0,0.024796800315380098
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,2,128,1,float16,float16,0,0.02414399981498718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,2,128,1,float16,fp8,0,0.02505280077457428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,2,128,1,fp8,fp8,0,0.025216001272201537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,4,128,1,float16,float16,0,0.024158400297164918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,4,128,1,float16,fp8,0,0.025128000974655153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,4,128,1,fp8,fp8,0,0.02516320049762726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,8,128,1,float16,float16,0,0.024563199281692503
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,8,128,1,float16,fp8,0,0.025252801179885865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,8,128,1,fp8,fp8,0,0.025121599435806274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,1,128,1,float16,float16,0,0.7448895931243896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,1,128,1,float16,fp8,0,0.8133919715881348
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,1,128,1,fp8,fp8,0,0.8137359619140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,2,128,1,float16,fp8,0,0.8468064308166504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,2,128,1,float16,float16,0,0.7775040149688721
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,2,128,1,fp8,fp8,0,0.8457167625427247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,4,128,1,float16,float16,0,0.848151969909668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,4,128,1,float16,fp8,0,0.9266112327575684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,4,128,1,fp8,fp8,0,0.9185680389404297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,8,128,1,float16,float16,0,1.0039584159851074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,1,128,1,float16,float16,0,0.3876703977584839
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,8,128,1,float16,fp8,0,1.085097599029541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,8,128,1,fp8,fp8,0,1.0824319839477539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,1,128,1,float16,fp8,0,0.4156816005706787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,1,128,1,fp8,fp8,0,0.41907200813293455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,2,128,1,float16,float16,0,0.40354080200195314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,64,128,1,float16,float16,0,1.6185792922973632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,64,128,1,float16,fp8,0,1.5752544403076172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,64,128,1,fp8,fp8,0,1.5751296043395997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,2,128,1,float16,fp8,0,0.43654718399047854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,2,128,1,fp8,fp8,0,0.4348015785217285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,4,128,1,float16,float16,0,0.43425760269165037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,4,128,1,float16,fp8,0,0.4739391803741455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,4,128,1,fp8,fp8,0,0.4761360168457031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,8,128,1,float16,float16,0,0.5107888221740723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,8,128,1,float16,fp8,0,0.5493072032928467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,8,128,1,fp8,fp8,0,0.5496928215026855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,1,128,1,float16,float16,0,0.2026671886444092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,64,128,1,float16,float16,0,0.8157903671264648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,64,128,1,float16,fp8,0,0.801638412475586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,64,128,1,fp8,fp8,0,0.7955632209777832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,1,128,1,float16,fp8,0,0.219815993309021
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,1,128,1,fp8,fp8,0,0.22011520862579345
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,2,128,1,float16,float16,0,0.2124880075454712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,2,128,1,float16,fp8,0,0.2288304090499878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,2,128,1,fp8,fp8,0,0.22810239791870118
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,4,128,1,float16,float16,0,0.22971038818359374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,4,128,1,float16,fp8,0,0.24678080081939696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,4,128,1,fp8,fp8,0,0.24694559574127198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,8,128,1,float16,float16,0,0.26670238971710203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,8,128,1,float16,fp8,0,0.28483679294586184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,8,128,1,fp8,fp8,0,0.28423519134521485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,64,128,1,float16,float16,0,0.4239984035491943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,1,128,1,fp8,fp8,0,0.12190719842910766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,64,128,1,float16,fp8,0,0.4136655807495117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,1,128,1,float16,float16,0,0.11225919723510742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,64,128,1,fp8,fp8,0,0.4098320007324219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,1,128,1,float16,fp8,0,0.12097439765930176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,2,128,1,float16,float16,0,0.11797920465469361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,2,128,1,float16,fp8,0,0.12754080295562745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,2,128,1,fp8,fp8,0,0.12749760150909423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,4,128,1,float16,float16,0,0.12723679542541505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,4,128,1,float16,fp8,0,0.13489439487457275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,64,128,1,float16,float16,0,0.22630879878997803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,4,128,1,fp8,fp8,0,0.1359120011329651
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,8,128,1,float16,float16,0,0.14613280296325684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,8,128,1,float16,fp8,0,0.15441759824752807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,8,128,1,fp8,fp8,0,0.1546512007713318
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,64,128,1,float16,fp8,0,0.21581120491027833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,1,128,1,float16,float16,0,0.06963199973106385
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,64,128,1,fp8,fp8,0,0.2146912097930908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,1,128,1,float16,fp8,0,0.07323520183563233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,1,128,1,fp8,fp8,0,0.07309600114822387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,2,128,1,float16,float16,0,0.0706928014755249
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,2,128,1,float16,fp8,0,0.07349439859390258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,2,128,1,fp8,fp8,0,0.07343199849128723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,4,128,1,float16,float16,0,0.0750656008720398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,4,128,1,float16,fp8,0,0.07904480099678039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,4,128,1,fp8,fp8,0,0.07893760204315185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,8,128,1,float16,float16,0,0.08448160290718079
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,8,128,1,float16,fp8,0,0.08700479865074158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,8,128,1,fp8,fp8,0,0.08658239841461182
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,64,128,1,float16,float16,0,0.12370400428771973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,64,128,1,float16,fp8,0,0.1164944052696228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,64,128,1,fp8,fp8,0,0.11553599834442138
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,1,128,1,float16,float16,0,0.041396799683570865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,1,128,1,float16,fp8,0,0.04178400039672851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,4,128,1,float16,fp8,0,0.041982400417327884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,1,128,1,fp8,fp8,0,0.041875201463699344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,2,128,1,float16,float16,0,0.04190399944782257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,2,128,1,float16,fp8,0,0.041017600893974306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,2,128,1,fp8,fp8,0,0.04070720076560974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,4,128,1,float16,float16,0,0.045465600490570066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,4,128,1,fp8,fp8,0,0.041966399550437926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,8,128,1,float16,float16,0,0.052723199129104614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,8,128,1,float16,fp8,0,0.04760479927062988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,8,128,1,fp8,fp8,0,0.04856640100479126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,64,128,1,float16,float16,0,0.07026559710502625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,64,128,1,float16,fp8,0,0.060102397203445436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,64,128,1,fp8,fp8,0,0.05978879928588867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,1,128,1,float16,float16,0,0.02874239981174469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,1,128,1,float16,fp8,0,0.029788801074028017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,1,128,1,fp8,fp8,0,0.029926401376724244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,2,128,1,float16,float16,0,0.028424000740051268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,2,128,1,float16,fp8,0,0.02959200143814087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,2,128,1,fp8,fp8,0,0.02969439923763275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,4,128,1,float16,float16,0,0.02887200117111206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,4,128,1,float16,fp8,0,0.02975519895553589
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,4,128,1,fp8,fp8,0,0.029974400997161865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,8,128,1,float16,float16,0,0.02969599962234497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,8,128,1,float16,fp8,0,0.03065119981765747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,8,128,1,fp8,fp8,0,0.030551999807357788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,64,128,1,float16,float16,0,0.036473599076271054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,64,128,1,float16,fp8,0,0.03578400015830994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,64,128,1,fp8,fp8,0,0.03584479987621307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,1,128,1,float16,float16,0,0.022603200376033784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,1,128,1,float16,fp8,0,0.023438400030136107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,1,128,1,fp8,fp8,0,0.023363199830055238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,2,128,1,float16,float16,0,0.02298559993505478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,2,128,1,float16,fp8,0,0.02332639992237091
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,2,128,1,fp8,fp8,0,0.02341119945049286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,4,128,1,float16,float16,0,0.02279680073261261
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,4,128,1,float16,fp8,0,0.023427200317382813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,4,128,1,fp8,fp8,0,0.023396800458431243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,8,128,1,float16,float16,0,0.02316479980945587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,8,128,1,float16,fp8,0,0.023556800186634065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,8,128,1,fp8,fp8,0,0.023659199476242065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,64,128,1,float16,float16,0,0.02723039984703064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,64,128,1,float16,fp8,0,0.027987200021743774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,64,128,1,fp8,fp8,0,0.027851200103759764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,1,128,1,float16,float16,0,0.02182080000638962
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,1,128,1,float16,fp8,0,0.022379200160503387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,1,128,1,fp8,fp8,0,0.022257600724697114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,2,128,1,float16,float16,0,0.021750399470329286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,2,128,1,float16,fp8,0,0.02239679992198944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,4,128,1,float16,float16,0,0.021855999529361726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,2,128,1,fp8,fp8,0,0.02223999947309494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,4,128,1,float16,fp8,0,0.022489599883556366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,4,128,1,fp8,fp8,0,0.022393600642681123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,8,128,1,float16,float16,0,0.02194720059633255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,8,128,1,float16,fp8,0,0.02263360023498535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,8,128,1,fp8,fp8,0,0.022567999362945557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,1,128,1,float16,float16,0,0.38714239597320554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,1,128,1,float16,fp8,0,0.4212031841278076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,1,128,1,fp8,fp8,0,0.41803679466247556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,2,128,1,float16,float16,0,0.406276798248291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,2,128,1,float16,fp8,0,0.4380623817443848
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,2,128,1,fp8,fp8,0,0.43912320137023925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,4,128,1,float16,float16,0,0.4407599925994873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,4,128,1,float16,fp8,0,0.47338080406188965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,4,128,1,fp8,fp8,0,0.47447519302368163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,8,128,1,float16,float16,0,0.5164735794067383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,8,128,1,fp8,fp8,0,0.5503952026367187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,8,128,1,float16,fp8,0,0.5510303974151611
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,1,128,1,float16,float16,0,0.20660159587860108
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,1,128,1,float16,fp8,0,0.2214400053024292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,64,128,1,float16,float16,0,0.9146847724914551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,1,128,1,fp8,fp8,0,0.22181599140167235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,64,128,1,float16,fp8,0,0.9416943550109863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,64,128,1,fp8,fp8,0,0.9411312103271484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,2,128,1,float16,float16,0,0.21369760036468505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,2,128,1,float16,fp8,0,0.23029921054840088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,2,128,1,fp8,fp8,0,0.22978560924530028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,4,128,1,float16,float16,0,0.2320240020751953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,8,128,1,float16,fp8,0,0.2855151891708374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,4,128,1,float16,fp8,0,0.24785120487213136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,4,128,1,fp8,fp8,0,0.24726080894470215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,8,128,1,float16,float16,0,0.2656415939331055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,8,128,1,fp8,fp8,0,0.28485119342803955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,1,128,1,fp8,fp8,0,0.1222607970237732
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,64,128,1,float16,float16,0,0.4682640075683594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,1,128,1,float16,float16,0,0.11535359621047973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,64,128,1,float16,fp8,0,0.4832479953765869
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,64,128,1,fp8,fp8,0,0.48290238380432127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,1,128,1,float16,fp8,0,0.12326079607009888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,2,128,1,float16,float16,0,0.11965759992599487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,2,128,1,float16,fp8,0,0.12893120050430298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,2,128,1,fp8,fp8,0,0.12904640436172485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,4,128,1,float16,float16,0,0.1278048038482666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,4,128,1,float16,fp8,0,0.1366752028465271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,4,128,1,fp8,fp8,0,0.1366160035133362
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,8,128,1,float16,float16,0,0.14675840139389038
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,8,128,1,fp8,fp8,0,0.15488320589065552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,8,128,1,float16,fp8,0,0.15538079738616944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,64,128,1,float16,float16,0,0.24777119159698485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,64,128,1,float16,fp8,0,0.25050721168518064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,1,128,1,float16,float16,0,0.07014240026473999
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,64,128,1,fp8,fp8,0,0.2513567924499512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,1,128,1,float16,fp8,0,0.07372959852218627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,1,128,1,fp8,fp8,0,0.07343519926071167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,2,128,1,float16,float16,0,0.07041440010070801
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,2,128,1,float16,fp8,0,0.07447199821472168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,2,128,1,fp8,fp8,0,0.07396640181541443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,4,128,1,float16,float16,0,0.07636640071868897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,4,128,1,float16,fp8,0,0.07944480180740357
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,4,128,1,fp8,fp8,0,0.07940800189971924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,8,128,1,float16,float16,0,0.0847648024559021
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,8,128,1,float16,fp8,0,0.08695679903030396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,8,128,1,fp8,fp8,0,0.08719040155410766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,64,128,1,float16,float16,0,0.13623839616775513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,64,128,1,float16,fp8,0,0.13349759578704834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,64,128,1,fp8,fp8,0,0.13402080535888672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,1,128,1,float16,float16,0,0.04005599915981293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,1,128,1,float16,fp8,0,0.040174400806427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,1,128,1,fp8,fp8,0,0.04057759940624237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,2,128,1,float16,float16,0,0.04159519970417023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,2,128,1,float16,fp8,0,0.0406031996011734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,2,128,1,fp8,fp8,0,0.04152159988880157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,4,128,1,float16,float16,0,0.04287840127944946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,64,128,1,float16,float16,0,0.07766720056533813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,4,128,1,float16,fp8,0,0.04149599969387054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,64,128,1,fp8,fp8,0,0.06774399876594543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,1,128,1,float16,float16,0,0.028358399868011475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,4,128,1,fp8,fp8,0,0.04143519997596741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,8,128,1,float16,float16,0,0.05278720259666443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,8,128,1,float16,fp8,0,0.04704320132732391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,8,128,1,fp8,fp8,0,0.04668639898300171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,64,128,1,float16,fp8,0,0.06949759721755981
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,1,128,1,float16,fp8,0,0.029841598868370057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,1,128,1,fp8,fp8,0,0.029872000217437744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,2,128,1,float16,float16,0,0.028360000252723692
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,2,128,1,float16,fp8,0,0.029864001274108886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,2,128,1,fp8,fp8,0,0.029553601145744325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,4,128,1,float16,float16,0,0.028548800945281984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,4,128,1,float16,fp8,0,0.02974559962749481
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,4,128,1,fp8,fp8,0,0.029708799719810487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,8,128,1,float16,float16,0,0.02930560111999512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,8,128,1,float16,fp8,0,0.030774399638175964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,8,128,1,fp8,fp8,0,0.030535998940467834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,64,128,1,float16,float16,0,0.03955360054969788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,64,128,1,float16,fp8,0,0.0397024005651474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,64,128,1,fp8,fp8,0,0.03994239866733551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,1,128,1,float16,float16,0,0.022422400116920472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,1,128,1,float16,fp8,0,0.0235615998506546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,1,128,1,fp8,fp8,0,0.02319519966840744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,2,128,1,float16,float16,0,0.022571200132369997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,2,128,1,float16,fp8,0,0.02329760044813156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,2,128,1,fp8,fp8,0,0.023585599660873414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,4,128,1,float16,float16,0,0.02258719950914383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,64,128,1,float16,float16,0,0.02686080038547516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,4,128,1,float16,fp8,0,0.02346719950437546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,4,128,1,fp8,fp8,0,0.02311840057373047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,8,128,1,float16,float16,0,0.022833600640296936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,8,128,1,float16,fp8,0,0.02364159971475601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,8,128,1,fp8,fp8,0,0.02330880016088486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,64,128,1,fp8,fp8,0,0.02767519950866699
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,64,128,1,float16,fp8,0,0.027452799677848815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,1,128,1,float16,float16,0,0.021561600267887115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,1,128,1,float16,fp8,0,0.022551999986171724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,1,128,1,fp8,fp8,0,0.022252799570560457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,2,128,1,float16,float16,0,0.021803200244903564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,2,128,1,float16,fp8,0,0.022308799624443054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,2,128,1,fp8,fp8,0,0.022511999309062957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,4,128,1,float16,float16,0,0.021731199324131013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,4,128,1,float16,fp8,0,0.02248159945011139
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,4,128,1,fp8,fp8,0,0.022383999824523926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,8,128,1,float16,float16,0,0.02168159931898117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,8,128,1,float16,fp8,0,0.02272160053253174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,8,128,1,fp8,fp8,0,0.022401599586009978
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,64,128,1,float16,float16,0,0.02222079932689667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,64,128,1,float16,fp8,0,0.02293439954519272
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,64,128,1,fp8,fp8,0,0.023177599906921385
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,1,128,1,float16,float16,0,0.020880000293254854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,1,128,1,float16,fp8,0,0.022014400362968443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,1,128,1,fp8,fp8,0,0.0220208004117012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,2,128,1,float16,float16,0,0.021059200167655945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,2,128,1,float16,fp8,0,0.022312000393867493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,2,128,1,fp8,fp8,0,0.02200479954481125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,4,128,1,float16,float16,0,0.021465599536895752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,4,128,1,float16,fp8,0,0.022006399929523468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,4,128,1,fp8,fp8,0,0.022041599452495574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,8,128,1,float16,float16,0,0.021350400149822236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,1,128,1,fp8,fp8,0,0.22280640602111818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,8,128,1,float16,fp8,0,0.021987199783325195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,8,128,1,fp8,fp8,0,0.022278399765491487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,1,128,1,float16,float16,0,0.20635199546813965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,1,128,1,float16,fp8,0,0.22268800735473632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,2,128,1,float16,float16,0,0.21400959491729737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,2,128,1,float16,fp8,0,0.2305056095123291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,2,128,1,fp8,fp8,0,0.23004639148712158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,4,128,1,float16,float16,0,0.23145599365234376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,4,128,1,float16,fp8,0,0.2481071949005127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,4,128,1,fp8,fp8,0,0.24926719665527344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,8,128,1,float16,float16,0,0.2694511890411377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,8,128,1,float16,fp8,0,0.2844111919403076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,8,128,1,fp8,fp8,0,0.2841968059539795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,64,128,1,float16,float16,0,0.5869679927825928
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,64,128,1,float16,fp8,0,0.614358377456665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,1,128,1,float16,float16,0,0.11488319635391235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,64,128,1,fp8,fp8,0,0.6125520229339599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,1,128,1,float16,fp8,0,0.1244271993637085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,1,128,1,fp8,fp8,0,0.12341439723968506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,2,128,1,float16,float16,0,0.12079839706420899
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,8,128,1,float16,float16,0,0.14820159673690797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,2,128,1,float16,fp8,0,0.12924799919128419
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,2,128,1,fp8,fp8,0,0.12929760217666625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,4,128,1,float16,float16,0,0.12700799703598023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,4,128,1,float16,fp8,0,0.13686879873275756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,4,128,1,fp8,fp8,0,0.13657439947128297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,8,128,1,float16,fp8,0,0.15512479543685914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,8,128,1,fp8,fp8,0,0.1548751950263977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,64,128,1,float16,float16,0,0.3057152032852173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,64,128,1,float16,fp8,0,0.3190079927444458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,1,128,1,float16,float16,0,0.0718608021736145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,64,128,1,fp8,fp8,0,0.32080318927764895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,1,128,1,float16,fp8,0,0.07431520223617553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,1,128,1,fp8,fp8,0,0.0741919994354248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,2,128,1,float16,float16,0,0.07243199944496155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,2,128,1,float16,fp8,0,0.07523040175437927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,2,128,1,fp8,fp8,0,0.07472959756851197
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,4,128,1,float16,float16,0,0.0773904025554657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,4,128,1,float16,fp8,0,0.08111839890480041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,4,128,1,fp8,fp8,0,0.08083840012550354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,8,128,1,float16,float16,0,0.08570079803466797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,8,128,1,float16,fp8,0,0.08801760077476502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,8,128,1,fp8,fp8,0,0.08807839751243592
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,64,128,1,float16,float16,0,0.1666208028793335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,64,128,1,float16,fp8,0,0.17110079526901245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,1,128,1,float16,float16,0,0.04054400026798248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,64,128,1,fp8,fp8,0,0.17066240310668945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,1,128,1,float16,fp8,0,0.0417167991399765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,1,128,1,fp8,fp8,0,0.040468800067901614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,2,128,1,float16,float16,0,0.04160960018634796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,2,128,1,float16,fp8,0,0.041412800550460815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,2,128,1,fp8,fp8,0,0.04110400080680847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,4,128,1,float16,float16,0,0.044784000515937804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,4,128,1,float16,fp8,0,0.042153599858284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,4,128,1,fp8,fp8,0,0.042340800166130066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,8,128,1,float16,float16,0,0.052830398082733154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,8,128,1,float16,fp8,0,0.04742400050163269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,8,128,1,fp8,fp8,0,0.04738720059394837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,64,128,1,float16,float16,0,0.09259039759635926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,64,128,1,float16,fp8,0,0.08650720119476318
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,64,128,1,fp8,fp8,0,0.08740000128746032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,1,128,1,float16,float16,0,0.028220799565315247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,1,128,1,float16,fp8,0,0.029912000894546507
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,1,128,1,fp8,fp8,0,0.029843199253082275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,2,128,1,float16,float16,0,0.028436800837516783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,2,128,1,float16,fp8,0,0.02959679961204529
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,2,128,1,fp8,fp8,0,0.02943199872970581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,8,128,1,fp8,fp8,0,0.03043839931488037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,4,128,1,float16,float16,0,0.028454399108886717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,64,128,1,float16,float16,0,0.04617919921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,4,128,1,float16,fp8,0,0.029867199063301087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,4,128,1,fp8,fp8,0,0.02985120117664337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,8,128,1,float16,float16,0,0.02951520085334778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,8,128,1,float16,fp8,0,0.030619201064109803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,64,128,1,float16,fp8,0,0.04864960014820099
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,64,128,1,fp8,fp8,0,0.048583999276161194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,1,128,1,float16,float16,0,0.02255360037088394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,1,128,1,float16,fp8,0,0.023511999845504762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,1,128,1,fp8,fp8,0,0.02346560060977936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,2,128,1,float16,float16,0,0.02274399995803833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,2,128,1,float16,fp8,0,0.023391999304294586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,2,128,1,fp8,fp8,0,0.02359039932489395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,4,128,1,float16,float16,0,0.022601599991321563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,4,128,1,float16,fp8,0,0.023287999629974365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,4,128,1,fp8,fp8,0,0.02346719950437546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,8,128,1,float16,float16,0,0.02273920029401779
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,8,128,1,float16,fp8,0,0.02377600073814392
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,8,128,1,fp8,fp8,0,0.02351039946079254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,64,128,1,float16,float16,0,0.03041599988937378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,64,128,1,float16,fp8,0,0.03179360032081604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,64,128,1,fp8,fp8,0,0.03193599879741669
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,1,128,1,float16,float16,0,0.02143999934196472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,1,128,1,float16,fp8,0,0.022280000150203705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,1,128,1,fp8,fp8,0,0.022270399332046508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,2,128,1,float16,float16,0,0.021566399931907655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,2,128,1,float16,fp8,0,0.022041599452495574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,2,128,1,fp8,fp8,0,0.022422400116920472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,4,128,1,float16,float16,0,0.021697600185871125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,4,128,1,float16,fp8,0,0.02280319929122925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,4,128,1,fp8,fp8,0,0.022694399952888487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,8,128,1,float16,float16,0,0.021753600239753722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,8,128,1,float16,fp8,0,0.022407999634742735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,8,128,1,fp8,fp8,0,0.02245599925518036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,64,128,1,float16,float16,0,0.022201600670814513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,64,128,1,float16,fp8,0,0.02322079986333847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,64,128,1,fp8,fp8,0,0.023185600340366364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,1,128,1,float16,float16,0,0.020947200059890748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,1,128,1,float16,fp8,0,0.021646399796009064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,1,128,1,fp8,fp8,0,0.0216048002243042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,2,128,1,float16,float16,0,0.02150239944458008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,2,128,1,float16,fp8,0,0.022407999634742735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,2,128,1,fp8,fp8,0,0.02160799950361252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,4,128,1,float16,float16,0,0.021555200219154358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,4,128,1,float16,fp8,0,0.02252320051193237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,4,128,1,fp8,fp8,0,0.02188960015773773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,8,128,1,float16,float16,0,0.021342399716377258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,8,128,1,float16,fp8,0,0.022065599262714387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,8,128,1,fp8,fp8,0,0.022120000422000886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,64,128,1,float16,float16,0,0.021422399580478667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,64,128,1,float16,fp8,0,0.021955199539661407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,64,128,1,fp8,fp8,0,0.02218559980392456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,1,128,1,float16,float16,0,0.020895999670028687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,1,128,1,float16,fp8,0,0.021214400231838227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,1,128,1,fp8,fp8,0,0.02134400010108948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,2,128,1,float16,float16,0,0.020587199926376344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,2,128,1,float16,fp8,0,0.021568000316619873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,2,128,1,fp8,fp8,0,0.021524800360202788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,4,128,1,float16,float16,0,0.020718400180339814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,4,128,1,float16,fp8,0,0.021529600024223328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,4,128,1,fp8,fp8,0,0.021747200191020964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,8,128,1,float16,float16,0,0.020931200683116914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,8,128,1,float16,fp8,0,0.021809600293636322
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,2,128,1,float16,fp8,0,0.1298815965652466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,8,128,1,fp8,fp8,0,0.021380800008773803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,1,128,1,float16,float16,0,0.11414079666137696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,1,128,1,float16,fp8,0,0.12456480264663697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,1,128,1,fp8,fp8,0,0.12510720491409302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,2,128,1,float16,float16,0,0.11956640481948852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,2,128,1,fp8,fp8,0,0.12970240116119386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,4,128,1,float16,float16,0,0.1295151948928833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,4,128,1,float16,fp8,0,0.1372032046318054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,4,128,1,fp8,fp8,0,0.13762240409851073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,8,128,1,float16,float16,0,0.1468575954437256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,8,128,1,float16,fp8,0,0.1559615969657898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,8,128,1,fp8,fp8,0,0.15614399909973145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,64,128,1,float16,float16,0,0.4257967948913574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,64,128,1,float16,fp8,0,0.4648399829864502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,1,128,1,float16,float16,0,0.07232319712638854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,1,128,1,float16,fp8,0,0.07515519857406616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,64,128,1,fp8,fp8,0,0.466153621673584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,1,128,1,fp8,fp8,0,0.07556800246238708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,4,128,1,fp8,fp8,0,0.08094239830970765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,2,128,1,float16,float16,0,0.0727504014968872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,2,128,1,float16,fp8,0,0.07560960054397584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,2,128,1,fp8,fp8,0,0.07569599747657776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,4,128,1,float16,float16,0,0.0781216025352478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,4,128,1,float16,fp8,0,0.08170400261878967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,8,128,1,float16,float16,0,0.08595359921455384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,8,128,1,float16,fp8,0,0.08875679969787598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,8,128,1,fp8,fp8,0,0.08877120018005372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,64,128,1,float16,float16,0,0.2234544038772583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,64,128,1,float16,fp8,0,0.24353439807891847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,1,128,1,float16,float16,0,0.04127359986305237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,64,128,1,fp8,fp8,0,0.24471681118011473
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,1,128,1,float16,fp8,0,0.04103519916534424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,1,128,1,fp8,fp8,0,0.04118559956550598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,2,128,1,float16,float16,0,0.042217600345611575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,2,128,1,float16,fp8,0,0.04114879965782166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,2,128,1,fp8,fp8,0,0.041201600432395936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,4,128,1,float16,float16,0,0.04524640142917633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,4,128,1,float16,fp8,0,0.04255039989948273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,64,128,1,float16,float16,0,0.12338559627532959
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,64,128,1,float16,fp8,0,0.12277120351791382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,4,128,1,fp8,fp8,0,0.042089599370956424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,8,128,1,float16,float16,0,0.05377920269966126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,8,128,1,float16,fp8,0,0.04819200038909912
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,8,128,1,fp8,fp8,0,0.04729120135307312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,1,128,1,float16,float16,0,0.028907200694084166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,64,128,1,fp8,fp8,0,0.12301440238952636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,1,128,1,float16,fp8,0,0.030124801397323608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,4,128,1,float16,fp8,0,0.030107200145721436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,1,128,1,fp8,fp8,0,0.03012320101261139
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,2,128,1,float16,float16,0,0.02882719933986664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,2,128,1,float16,fp8,0,0.030057600140571593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,2,128,1,fp8,fp8,0,0.03004319965839386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,4,128,1,float16,float16,0,0.02898080050945282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,4,128,1,fp8,fp8,0,0.030350399017333985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,8,128,1,float16,float16,0,0.029812800884246825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,8,128,1,float16,fp8,0,0.031070399284362792
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,8,128,1,fp8,fp8,0,0.03115360140800476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,64,128,1,float16,float16,0,0.061380797624588014
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,64,128,1,float16,fp8,0,0.0668175995349884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,64,128,1,fp8,fp8,0,0.06690080165863037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,1,128,1,float16,float16,0,0.022947199642658234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,1,128,1,float16,fp8,0,0.023447999358177186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,1,128,1,fp8,fp8,0,0.02354239970445633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,2,128,1,float16,float16,0,0.022947199642658234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,2,128,1,float16,fp8,0,0.02369759976863861
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,2,128,1,fp8,fp8,0,0.023384000360965728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,4,128,1,float16,float16,0,0.02285120040178299
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,4,128,1,float16,fp8,0,0.023555199801921844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,4,128,1,fp8,fp8,0,0.023612800240516662
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,8,128,1,float16,float16,0,0.02298559993505478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,1,128,1,float16,fp8,0,0.02229280024766922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,8,128,1,float16,fp8,0,0.0236272007226944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,8,128,1,fp8,fp8,0,0.023820799589157105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,64,128,1,float16,float16,0,0.03799520134925842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,64,128,1,float16,fp8,0,0.04105440080165863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,4,128,1,float16,float16,0,0.021817600727081297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,64,128,1,fp8,fp8,0,0.040915200114250184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,1,128,1,float16,float16,0,0.02156960070133209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,1,128,1,fp8,fp8,0,0.022625599801540375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,2,128,1,float16,float16,0,0.021740800142288207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,2,128,1,float16,fp8,0,0.022540800273418427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,2,128,1,fp8,fp8,0,0.022353599965572356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,4,128,1,float16,fp8,0,0.02276960015296936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,4,128,1,fp8,fp8,0,0.02266400009393692
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,8,128,1,float16,float16,0,0.021614399552345277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,8,128,1,float16,fp8,0,0.022720000147819518
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,8,128,1,fp8,fp8,0,0.022521600127220154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,2,128,1,float16,float16,0,0.0210207998752594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,64,128,1,float16,float16,0,0.026104000210762025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,64,128,1,float16,fp8,0,0.02778880000114441
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,64,128,1,fp8,fp8,0,0.027590399980545043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,1,128,1,float16,float16,0,0.021134400367736818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,1,128,1,float16,fp8,0,0.021731199324131013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,1,128,1,fp8,fp8,0,0.021559999883174898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,2,128,1,float16,fp8,0,0.02184319943189621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,2,128,1,fp8,fp8,0,0.02149759978055954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,4,128,1,float16,float16,0,0.021244800090789794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,4,128,1,float16,fp8,0,0.021967999637126923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,4,128,1,fp8,fp8,0,0.02229599952697754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,8,128,1,float16,float16,0,0.021423999965190888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,8,128,1,float16,fp8,0,0.022283199429512023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,8,128,1,fp8,fp8,0,0.021879999339580535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,64,128,1,float16,float16,0,0.02152319997549057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,64,128,1,float16,fp8,0,0.022193600237369538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,64,128,1,fp8,fp8,0,0.02232639938592911
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,1,128,1,float16,float16,0,0.021048000454902648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,1,128,1,float16,fp8,0,0.02147040069103241
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,1,128,1,fp8,fp8,0,0.021393600106239318
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,2,128,1,float16,float16,0,0.020707200467586517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,2,128,1,float16,fp8,0,0.02165600061416626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,2,128,1,fp8,fp8,0,0.02143840044736862
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,4,128,1,float16,float16,0,0.020771199464797975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,4,128,1,float16,fp8,0,0.021631999313831328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,4,128,1,fp8,fp8,0,0.021932800114154816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,8,128,1,float16,float16,0,0.020942400395870208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,8,128,1,float16,fp8,0,0.021831999719142913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,8,128,1,fp8,fp8,0,0.021638399362564086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,64,128,1,float16,float16,0,0.0209647998213768
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,64,128,1,float16,fp8,0,0.021592000126838685
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,64,128,1,fp8,fp8,0,0.021862399578094483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,1,128,1,float16,float16,0,0.020494399964809416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,1,128,1,float16,fp8,0,0.021564799547195434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,1,128,1,fp8,fp8,0,0.02139520049095154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,2,128,1,float16,float16,0,0.020721599459648132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,2,128,1,float16,fp8,0,0.021211199462413788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,2,128,1,fp8,fp8,0,0.021631999313831328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,4,128,1,float16,float16,0,0.020662400126457214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,4,128,1,float16,fp8,0,0.021408000588417055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,4,128,1,fp8,fp8,0,0.021291199326515197
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,8,128,1,float16,float16,0,0.020713600516319274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,8,128,1,float16,fp8,0,0.021619200706481934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,8,128,1,fp8,fp8,0,0.021359999477863312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,1,128,1,fp8,fp8,0,38.657736206054686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,1,128,1,float16,fp8,0,39.09449462890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,2,128,1,float16,fp8,0,38.354278564453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,2,128,1,fp8,fp8,0,38.35318603515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,1,128,1,float16,float16,0,46.317874145507815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,2,128,1,float16,float16,0,46.75351257324219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,4,128,1,float16,float16,0,46.44600830078125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,4,128,1,float16,fp8,0,38.67091674804688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,48,128,1,fp8,fp8,0,21.792750549316406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,48,128,1,float16,fp8,0,22.72039337158203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,48,128,1,float16,float16,0,26.091119384765626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,1,128,1,float16,float16,0,22.22715606689453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,4,128,1,fp8,fp8,0,39.37763977050781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,8,128,1,float16,fp8,0,39.54453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,8,128,1,fp8,fp8,0,39.401876831054686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,8,128,1,float16,float16,0,47.888037109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,1,128,1,float16,fp8,0,19.70091552734375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,1,128,1,fp8,fp8,0,19.594969177246092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,2,128,1,float16,fp8,0,19.701002502441405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,2,128,1,float16,float16,0,22.762240600585937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,2,128,1,fp8,fp8,0,19.00543212890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,4,128,1,float16,fp8,0,19.162530517578126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,4,128,1,float16,float16,0,22.741473388671874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,4,128,1,fp8,fp8,0,19.252658081054687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,48,128,1,float16,float16,0,13.166136169433594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,48,128,1,float16,fp8,0,11.288257598876953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,8,128,1,float16,fp8,0,20.484394836425782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,48,128,1,fp8,fp8,0,11.028076934814454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,8,128,1,float16,float16,0,22.995167541503907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,8,128,1,fp8,fp8,0,19.58131103515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,1,128,1,float16,float16,0,11.610422515869141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,1,128,1,float16,fp8,0,10.256972503662109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,1,128,1,fp8,fp8,0,9.803302764892578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,2,128,1,float16,fp8,0,10.020889282226562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,2,128,1,fp8,fp8,0,10.23134536743164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,2,128,1,float16,float16,0,12.13609619140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,4,128,1,float16,fp8,0,9.687276458740234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,4,128,1,float16,float16,0,11.804312133789063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,4,128,1,fp8,fp8,0,9.991169738769532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,48,128,1,float16,fp8,0,5.498889541625976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,48,128,1,float16,float16,0,6.740436553955078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,8,128,1,float16,fp8,0,9.721405029296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,8,128,1,float16,float16,0,12.296949005126953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,48,128,1,fp8,fp8,0,5.800070571899414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,1,128,1,float16,fp8,0,4.721393585205078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,1,128,1,float16,float16,0,5.462900924682617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,8,128,1,fp8,fp8,0,10.566824340820313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,1,128,1,fp8,fp8,0,4.654131317138672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,2,128,1,float16,fp8,0,4.683179092407227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,2,128,1,fp8,fp8,0,4.752094268798828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,2,128,1,float16,float16,0,5.607732772827148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,4,128,1,float16,fp8,0,4.7377471923828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,4,128,1,float16,float16,0,5.9624065399169925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,4,128,1,fp8,fp8,0,4.830673599243164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,8,128,1,float16,fp8,0,4.842897415161133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,8,128,1,float16,float16,0,6.258628845214844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,8,128,1,fp8,fp8,0,4.8270526885986325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,1,128,1,float16,fp8,0,21.835614013671876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,1,128,1,fp8,fp8,0,22.2985595703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,2,128,1,float16,fp8,0,21.900711059570312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,2,128,1,fp8,fp8,0,23.043006896972656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,1,128,1,float16,float16,0,26.572344970703124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,2,128,1,float16,float16,0,25.767227172851562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,4,128,1,float16,fp8,0,22.741835021972655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,4,128,1,float16,float16,0,26.959915161132812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,48,128,1,fp8,fp8,0,13.113874816894532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,48,128,1,float16,fp8,0,13.439408874511718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,48,128,1,float16,float16,0,15.772209167480469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,1,128,1,float16,float16,0,13.275160217285157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,4,128,1,fp8,fp8,0,22.415625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,8,128,1,float16,fp8,0,22.47625732421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,8,128,1,fp8,fp8,0,22.530972290039063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,8,128,1,float16,float16,0,26.780374145507814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,1,128,1,float16,fp8,0,11.174406433105469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,1,128,1,fp8,fp8,0,10.833806610107422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,2,128,1,float16,float16,0,13.167692565917969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,2,128,1,fp8,fp8,0,10.849737548828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,2,128,1,float16,fp8,0,11.423033905029296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,4,128,1,float16,float16,0,13.169192504882812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,4,128,1,float16,fp8,0,11.629946899414062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,4,128,1,fp8,fp8,0,11.305249786376953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,48,128,1,float16,fp8,0,6.808708953857422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,8,128,1,float16,fp8,0,11.320355224609376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,48,128,1,float16,float16,0,8.372608184814453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,8,128,1,float16,float16,0,13.778294372558594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,48,128,1,fp8,fp8,0,6.706180572509766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,1,128,1,float16,float16,0,6.482459259033203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,1,128,1,float16,fp8,0,5.595315170288086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,8,128,1,fp8,fp8,0,11.789064025878906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,1,128,1,fp8,fp8,0,5.544043350219726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,2,128,1,float16,fp8,0,5.479422378540039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,2,128,1,float16,float16,0,6.607839965820313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,2,128,1,fp8,fp8,0,5.350883102416992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,4,128,1,float16,float16,0,6.7975715637207035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,4,128,1,float16,fp8,0,5.461396789550781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,4,128,1,fp8,fp8,0,5.5558929443359375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,48,128,1,float16,float16,0,4.0773967742919925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,8,128,1,float16,float16,0,7.1096046447753904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,48,128,1,float16,fp8,0,3.2959823608398438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,8,128,1,float16,fp8,0,6.127206420898437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,8,128,1,fp8,fp8,0,5.578023910522461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,1,128,1,float16,float16,0,3.1718687057495116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,48,128,1,fp8,fp8,0,3.4247535705566405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,1,128,1,float16,fp8,0,2.769099235534668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,1,128,1,fp8,fp8,0,2.7638431549072267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,2,128,1,float16,fp8,0,2.867812728881836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,2,128,1,float16,float16,0,3.204510498046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,2,128,1,fp8,fp8,0,2.691619110107422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,4,128,1,float16,fp8,0,2.698776054382324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,4,128,1,float16,float16,0,3.1552783966064455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,4,128,1,fp8,fp8,0,2.72128963470459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,8,128,1,float16,fp8,0,2.938929557800293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,8,128,1,float16,float16,0,3.301670455932617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,8,128,1,fp8,fp8,0,2.8970016479492187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,1,128,1,float16,fp8,0,14.865121459960937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,1,128,1,fp8,fp8,0,15.587713623046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,2,128,1,float16,fp8,0,15.319129943847656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,1,128,1,float16,float16,0,18.25074462890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,2,128,1,fp8,fp8,0,15.930613708496093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,4,128,1,float16,fp8,0,15.678590393066406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,2,128,1,float16,float16,0,17.69214782714844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,4,128,1,float16,float16,0,18.77674255371094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,48,128,1,float16,fp8,0,9.699342346191406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,48,128,1,fp8,fp8,0,9.533074951171875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,48,128,1,float16,float16,0,11.48491973876953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,1,128,1,float16,float16,0,9.274483489990235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,4,128,1,fp8,fp8,0,15.738743591308594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,8,128,1,float16,fp8,0,16.394659423828124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,8,128,1,fp8,fp8,0,16.11883544921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,8,128,1,float16,float16,0,19.04197540283203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,1,128,1,float16,fp8,0,7.744312286376953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,1,128,1,fp8,fp8,0,8.03145751953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,2,128,1,float16,fp8,0,7.888272094726562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,2,128,1,fp8,fp8,0,7.837083435058593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,2,128,1,float16,float16,0,9.331478118896484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,4,128,1,float16,fp8,0,7.688835144042969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,4,128,1,float16,float16,0,9.282855987548828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,4,128,1,fp8,fp8,0,7.6675262451171875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,48,128,1,float16,float16,0,5.439566421508789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,48,128,1,float16,fp8,0,5.067119979858399
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,8,128,1,float16,fp8,0,8.278864288330078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,8,128,1,float16,float16,0,9.522217559814454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,8,128,1,fp8,fp8,0,7.955633544921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,1,128,1,float16,float16,0,4.366646575927734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,48,128,1,fp8,fp8,0,4.7935535430908205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,1,128,1,float16,fp8,0,3.80787353515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,1,128,1,fp8,fp8,0,3.8795169830322265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,2,128,1,float16,fp8,0,3.7740463256835937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,2,128,1,float16,float16,0,4.9032958984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,2,128,1,fp8,fp8,0,3.8142318725585938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,4,128,1,float16,fp8,0,3.8143646240234377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,4,128,1,float16,float16,0,4.464136123657227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,4,128,1,fp8,fp8,0,3.8489471435546876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,8,128,1,float16,float16,0,4.827568054199219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,48,128,1,float16,fp8,0,2.4139184951782227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,48,128,1,float16,float16,0,2.7441631317138673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,8,128,1,float16,fp8,0,3.8999073028564455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,8,128,1,fp8,fp8,0,3.904734420776367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,1,128,1,float16,fp8,0,1.8983392715454102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,48,128,1,fp8,fp8,0,2.4131328582763674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,1,128,1,float16,float16,0,2.5258432388305665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,1,128,1,fp8,fp8,0,1.9272575378417969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,2,128,1,float16,float16,0,2.2051664352416993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,2,128,1,fp8,fp8,0,1.8947423934936523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,2,128,1,float16,fp8,0,2.004640007019043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,4,128,1,float16,fp8,0,1.9269872665405274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,4,128,1,float16,float16,0,2.254867172241211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,4,128,1,fp8,fp8,0,1.9156431198120116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,8,128,1,float16,float16,0,2.4416303634643555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,8,128,1,float16,fp8,0,1.97259521484375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,8,128,1,fp8,fp8,0,2.1624576568603517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,1,128,1,fp8,fp8,0,19.980979919433594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,2,128,1,float16,fp8,0,20.513796997070312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,1,128,1,float16,fp8,0,21.120594787597657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,2,128,1,fp8,fp8,0,20.7179443359375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,4,128,1,float16,fp8,0,20.292538452148438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,1,128,1,float16,float16,0,23.802456665039063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,2,128,1,float16,float16,0,24.411483764648438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,4,128,1,float16,float16,0,24.118069458007813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,48,128,1,float16,fp8,0,13.012539672851563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,1,128,1,float16,float16,0,12.045381164550781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,48,128,1,fp8,fp8,0,13.06063232421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,48,128,1,float16,float16,0,15.561129760742187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,4,128,1,fp8,fp8,0,20.68214111328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,8,128,1,fp8,fp8,0,21.488543701171874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,8,128,1,float16,fp8,0,22.653294372558594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,8,128,1,float16,float16,0,24.7700439453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,1,128,1,float16,fp8,0,9.701006317138672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,1,128,1,fp8,fp8,0,9.872796630859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,2,128,1,float16,fp8,0,9.80658416748047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,2,128,1,float16,float16,0,11.90030746459961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,2,128,1,fp8,fp8,0,10.075740814208984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,4,128,1,float16,fp8,0,10.987630462646484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,4,128,1,fp8,fp8,0,10.2332275390625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,4,128,1,float16,float16,0,12.595613098144531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,48,128,1,float16,float16,0,7.789730834960937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,48,128,1,float16,fp8,0,6.541286468505859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,8,128,1,float16,fp8,0,10.565985870361327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,8,128,1,fp8,fp8,0,10.410137939453126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,8,128,1,float16,float16,0,12.841957092285156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,1,128,1,float16,fp8,0,4.926833724975586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,48,128,1,fp8,fp8,0,6.783907318115235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,1,128,1,float16,float16,0,6.075542449951172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,1,128,1,fp8,fp8,0,4.9714702606201175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,2,128,1,float16,fp8,0,5.086681747436524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,2,128,1,float16,float16,0,5.821441650390625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,2,128,1,fp8,fp8,0,4.980948638916016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,4,128,1,float16,fp8,0,5.3721473693847654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,4,128,1,fp8,fp8,0,5.177766418457031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,4,128,1,float16,float16,0,6.165494537353515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,48,128,1,float16,float16,0,3.729030227661133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,8,128,1,float16,float16,0,6.393348693847656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,48,128,1,float16,fp8,0,3.3152721405029295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,8,128,1,float16,fp8,0,5.308939361572266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,8,128,1,fp8,fp8,0,5.371649551391601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,1,128,1,float16,fp8,0,2.4950735092163088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,1,128,1,float16,float16,0,2.8474704742431642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,48,128,1,fp8,fp8,0,3.386872100830078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,1,128,1,fp8,fp8,0,2.5569503784179686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,2,128,1,float16,fp8,0,2.4692768096923827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,2,128,1,float16,float16,0,2.8401887893676756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,2,128,1,fp8,fp8,0,2.5557439804077147
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,4,128,1,float16,float16,0,3.0199920654296877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,4,128,1,float16,fp8,0,2.538265609741211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,4,128,1,fp8,fp8,0,2.7263023376464846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,8,128,1,float16,fp8,0,2.592305564880371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,8,128,1,float16,float16,0,3.4194416046142577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,48,128,1,float16,float16,0,1.874185562133789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,48,128,1,float16,fp8,0,1.676795196533203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,1,128,1,float16,float16,0,1.3740480422973633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,1,128,1,float16,fp8,0,1.264846420288086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,8,128,1,fp8,fp8,0,2.575494384765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,48,128,1,fp8,fp8,0,1.779952049255371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,1,128,1,fp8,fp8,0,1.237889575958252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,2,128,1,float16,float16,0,1.5815759658813477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,2,128,1,float16,fp8,0,1.362827205657959
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,2,128,1,fp8,fp8,0,1.2466064453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,4,128,1,float16,float16,0,1.4929327964782715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,4,128,1,float16,fp8,0,1.2668000221252442
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,4,128,1,fp8,fp8,0,1.262007999420166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,8,128,1,float16,float16,0,1.3970416069030762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,8,128,1,float16,fp8,0,1.292465591430664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,8,128,1,fp8,fp8,0,1.4363120079040528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,1,128,1,float16,fp8,0,11.249515533447266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,2,128,1,float16,fp8,0,11.428377532958985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,1,128,1,fp8,fp8,0,11.720196533203126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,2,128,1,fp8,fp8,0,11.511385345458985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,1,128,1,float16,float16,0,13.788121032714844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,2,128,1,float16,float16,0,13.313929748535156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,4,128,1,float16,fp8,0,12.777435302734375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,4,128,1,float16,float16,0,13.571405029296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,1,128,1,float16,float16,0,6.818907165527344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,48,128,1,float16,fp8,0,8.110208129882812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,48,128,1,float16,float16,0,9.485558319091798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,48,128,1,fp8,fp8,0,9.003177642822266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,4,128,1,fp8,fp8,0,11.945906829833984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,8,128,1,float16,fp8,0,12.531886291503906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,8,128,1,fp8,fp8,0,12.383412933349609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,8,128,1,float16,float16,0,14.830078125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,1,128,1,float16,fp8,0,5.694976043701172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,1,128,1,fp8,fp8,0,5.716843032836914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,2,128,1,fp8,fp8,0,5.919343948364258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,2,128,1,float16,fp8,0,6.493569946289062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,2,128,1,float16,float16,0,7.027436828613281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,4,128,1,float16,fp8,0,6.011881637573242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,4,128,1,float16,float16,0,7.174350738525391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,4,128,1,fp8,fp8,0,6.361524963378907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,8,128,1,float16,float16,0,7.384223937988281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,48,128,1,float16,fp8,0,4.21576156616211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,8,128,1,float16,fp8,0,6.225390243530273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,48,128,1,float16,float16,0,4.705118560791016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,1,128,1,float16,float16,0,3.3934513092041017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,48,128,1,fp8,fp8,0,4.157681655883789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,8,128,1,fp8,fp8,0,6.341316986083984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,1,128,1,float16,fp8,0,2.8837472915649416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,1,128,1,fp8,fp8,0,3.1230031967163088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,2,128,1,float16,fp8,0,2.9150495529174805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,2,128,1,float16,float16,0,3.238486480712891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,2,128,1,fp8,fp8,0,3.1301248550415037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,4,128,1,float16,fp8,0,2.9845983505249025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,4,128,1,fp8,fp8,0,2.9207855224609376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,4,128,1,float16,float16,0,3.4578529357910157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,8,128,1,float16,float16,0,3.431982421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,48,128,1,float16,fp8,0,2.072999954223633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,48,128,1,float16,float16,0,2.3062047958374023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,1,128,1,float16,float16,0,1.5565183639526368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,1,128,1,float16,fp8,0,1.5334511756896974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,8,128,1,float16,fp8,0,3.0250288009643556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,48,128,1,fp8,fp8,0,2.061379241943359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,8,128,1,fp8,fp8,0,3.064625549316406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,1,128,1,fp8,fp8,0,1.5907456398010253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,2,128,1,float16,fp8,0,1.5004351615905762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,2,128,1,float16,float16,0,1.86617431640625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,2,128,1,fp8,fp8,0,1.5911999702453614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,4,128,1,float16,fp8,0,1.473089599609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,4,128,1,float16,float16,0,1.6204448699951173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,4,128,1,fp8,fp8,0,1.4682479858398438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,8,128,1,float16,float16,0,1.6513248443603517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,8,128,1,float16,fp8,0,1.5392767906188964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,48,128,1,float16,float16,0,1.2280832290649415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,48,128,1,float16,fp8,0,1.077608013153076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,48,128,1,fp8,fp8,0,1.0391056060791015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,8,128,1,fp8,fp8,0,1.5551119804382325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,1,128,1,float16,float16,0,0.7732736110687256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,1,128,1,float16,fp8,0,0.8077407836914062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,1,128,1,fp8,fp8,0,0.7267712116241455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,2,128,1,float16,float16,0,0.8101183891296386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,2,128,1,float16,fp8,0,0.747436809539795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,2,128,1,fp8,fp8,0,0.75273118019104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,4,128,1,float16,float16,0,0.7991551876068115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,4,128,1,float16,fp8,0,0.7534080028533936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,4,128,1,fp8,fp8,0,0.7809311866760253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,8,128,1,float16,float16,0,0.8191472053527832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,8,128,1,float16,fp8,0,0.8587023735046386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,8,128,1,fp8,fp8,0,0.7768432140350342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,1,128,1,float16,fp8,0,10.62466049194336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,1,128,1,fp8,fp8,0,10.81637954711914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,2,128,1,float16,fp8,0,10.777294158935547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,1,128,1,float16,float16,0,12.302680206298827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,2,128,1,fp8,fp8,0,11.644222259521484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,2,128,1,float16,float16,0,12.578955078125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,4,128,1,float16,float16,0,12.899943542480468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,4,128,1,float16,fp8,0,11.14794692993164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,48,128,1,float16,fp8,0,8.57681884765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,48,128,1,float16,float16,0,9.67044448852539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,1,128,1,float16,float16,0,6.38109130859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,4,128,1,fp8,fp8,0,11.165131378173829
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,48,128,1,fp8,fp8,0,8.590252685546876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,8,128,1,float16,fp8,0,11.847840118408204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,8,128,1,fp8,fp8,0,11.630908966064453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,8,128,1,float16,float16,0,13.516241455078125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,1,128,1,float16,fp8,0,5.410801696777344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,1,128,1,fp8,fp8,0,5.711012649536133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,2,128,1,float16,fp8,0,5.41399040222168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,2,128,1,fp8,fp8,0,5.402374267578125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,2,128,1,float16,float16,0,6.323361587524414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,4,128,1,float16,fp8,0,5.666793441772461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,4,128,1,float16,float16,0,6.993777465820313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,4,128,1,fp8,fp8,0,5.718024063110351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,48,128,1,float16,fp8,0,4.348795318603516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,48,128,1,float16,float16,0,4.953651046752929
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,1,128,1,float16,fp8,0,2.685755157470703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,1,128,1,float16,float16,0,3.1992063522338867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,48,128,1,fp8,fp8,0,4.340366363525391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,8,128,1,float16,fp8,0,6.269879913330078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,8,128,1,fp8,fp8,0,6.040689468383789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,8,128,1,float16,float16,0,7.43335189819336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,1,128,1,fp8,fp8,0,2.7613840103149414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,2,128,1,float16,fp8,0,2.7410736083984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,2,128,1,fp8,fp8,0,2.8304895401000976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,2,128,1,float16,float16,0,3.3118896484375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,4,128,1,fp8,fp8,0,2.808380889892578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,4,128,1,float16,fp8,0,3.0427936553955077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,4,128,1,float16,float16,0,3.1523664474487303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,8,128,1,float16,float16,0,3.3880287170410157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,48,128,1,float16,float16,0,2.3384416580200194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,1,128,1,float16,fp8,0,1.4014224052429198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,8,128,1,fp8,fp8,0,2.939910316467285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,1,128,1,float16,float16,0,1.81900634765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,48,128,1,float16,fp8,0,2.368244743347168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,8,128,1,float16,fp8,0,3.3476959228515626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,1,128,1,fp8,fp8,0,1.4127375602722168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,48,128,1,fp8,fp8,0,2.18929443359375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,2,128,1,float16,float16,0,1.4795680046081543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,2,128,1,float16,fp8,0,1.427841567993164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,2,128,1,fp8,fp8,0,1.3741552352905273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,4,128,1,float16,fp8,0,1.407926368713379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,4,128,1,float16,float16,0,1.6310047149658202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,4,128,1,fp8,fp8,0,1.4240320205688477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,8,128,1,float16,float16,0,1.5697008132934571
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,8,128,1,float16,fp8,0,1.4755488395690919
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,48,128,1,float16,float16,0,1.1874272346496582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,48,128,1,float16,fp8,0,1.0973872184753417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,8,128,1,fp8,fp8,0,1.5809935569763183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,1,128,1,float16,float16,0,0.7870863914489746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,1,128,1,float16,fp8,0,0.6936912059783935
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,48,128,1,fp8,fp8,0,1.1098655700683593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,1,128,1,fp8,fp8,0,0.7619088172912598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,2,128,1,float16,float16,0,0.7365231990814209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,2,128,1,float16,fp8,0,0.7113296031951905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,2,128,1,fp8,fp8,0,0.6940464019775391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,4,128,1,float16,float16,0,0.7704815864562988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,4,128,1,float16,fp8,0,0.7799007892608643
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,4,128,1,fp8,fp8,0,0.7171264171600342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,8,128,1,float16,float16,0,0.7975615978240966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,8,128,1,float16,fp8,0,0.7889711856842041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,8,128,1,fp8,fp8,0,0.7530543804168701
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,48,128,1,float16,float16,0,0.6057456016540528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,48,128,1,float16,fp8,0,0.5615920066833496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,1,128,1,float16,float16,0,0.3919519901275635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,48,128,1,fp8,fp8,0,0.5756976127624511
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,1,128,1,float16,fp8,0,0.356873607635498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,4,128,1,float16,float16,0,0.3878015995025635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,1,128,1,fp8,fp8,0,0.39071519374847413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,2,128,1,float16,float16,0,0.3884111881256104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,2,128,1,float16,fp8,0,0.3729183912277222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,2,128,1,fp8,fp8,0,0.37298879623413084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,4,128,1,float16,fp8,0,0.3800607919692993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,4,128,1,fp8,fp8,0,0.378385591506958
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,8,128,1,float16,float16,0,0.409500789642334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,8,128,1,float16,fp8,0,0.3855423927307129
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,8,128,1,fp8,fp8,0,0.4020400047302246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,1,128,1,float16,fp8,0,6.36053581237793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,1,128,1,fp8,fp8,0,6.492095947265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,1,128,1,float16,float16,0,7.205881500244141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,2,128,1,float16,fp8,0,6.530168151855468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,2,128,1,fp8,fp8,0,6.527092742919922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,2,128,1,float16,float16,0,7.4113311767578125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,4,128,1,float16,float16,0,7.686086273193359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,4,128,1,float16,fp8,0,6.628672027587891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,48,128,1,float16,float16,0,6.1256561279296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,4,128,1,fp8,fp8,0,6.660123443603515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,48,128,1,float16,fp8,0,5.70750732421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,8,128,1,float16,fp8,0,7.0860435485839846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,8,128,1,fp8,fp8,0,7.10408935546875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,48,128,1,fp8,fp8,0,5.690510559082031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,8,128,1,float16,float16,0,8.18402099609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,1,128,1,float16,float16,0,3.6224433898925783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,1,128,1,float16,fp8,0,3.2379615783691404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,1,128,1,fp8,fp8,0,3.1593536376953124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,2,128,1,float16,float16,0,3.611521530151367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,2,128,1,float16,fp8,0,3.210030364990234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,2,128,1,fp8,fp8,0,3.2430896759033203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,4,128,1,float16,float16,0,3.74344482421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,4,128,1,float16,fp8,0,3.471638488769531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,4,128,1,fp8,fp8,0,3.3421550750732423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,8,128,1,float16,fp8,0,3.5523632049560545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,1,128,1,float16,float16,0,2.094068717956543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,48,128,1,float16,float16,0,3.03024959564209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,8,128,1,float16,float16,0,3.905556869506836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,48,128,1,float16,fp8,0,2.8547040939331056
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,8,128,1,fp8,fp8,0,3.6789905548095705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,48,128,1,fp8,fp8,0,3.024564743041992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,1,128,1,float16,fp8,0,1.596656036376953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,1,128,1,fp8,fp8,0,1.5938096046447754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,2,128,1,float16,float16,0,1.8774400711059571
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,2,128,1,float16,fp8,0,1.6134992599487306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,2,128,1,fp8,fp8,0,1.7955856323242188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,4,128,1,float16,float16,0,1.8337200164794922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,4,128,1,fp8,fp8,0,1.6867792129516601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,4,128,1,float16,fp8,0,1.704747200012207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,8,128,1,float16,float16,0,1.8753551483154296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,8,128,1,float16,fp8,0,1.7824176788330077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,1,128,1,float16,float16,0,0.8559424400329589
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,1,128,1,float16,fp8,0,0.8201727867126465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,48,128,1,float16,float16,0,1.5209919929504394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,48,128,1,float16,fp8,0,1.448953628540039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,8,128,1,fp8,fp8,0,1.925057601928711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,48,128,1,fp8,fp8,0,1.5393872261047363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,1,128,1,fp8,fp8,0,0.8336943626403809
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,2,128,1,float16,float16,0,0.8689567565917968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,2,128,1,float16,fp8,0,0.8359040260314942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,2,128,1,fp8,fp8,0,0.8367168426513671
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,4,128,1,float16,float16,0,0.8990400314331055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,4,128,1,float16,fp8,0,0.8550880432128907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,4,128,1,fp8,fp8,0,0.8622655868530273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,8,128,1,float16,float16,0,0.9685888290405273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,8,128,1,float16,fp8,0,0.9333312034606933
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,8,128,1,fp8,fp8,0,0.9090928077697754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,48,128,1,float16,float16,0,0.7891983985900879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,1,128,1,float16,float16,0,0.4377312183380127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,48,128,1,float16,fp8,0,0.7480175971984864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,1,128,1,float16,fp8,0,0.41717119216918946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,48,128,1,fp8,fp8,0,0.7280799865722656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,1,128,1,fp8,fp8,0,0.42559041976928713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,2,128,1,float16,float16,0,0.453220796585083
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,2,128,1,float16,fp8,0,0.4318367958068848
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,2,128,1,fp8,fp8,0,0.43422880172729494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,4,128,1,float16,float16,0,0.4654655933380127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,4,128,1,float16,fp8,0,0.44722881317138674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,4,128,1,fp8,fp8,0,0.44146242141723635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,8,128,1,float16,float16,0,0.48030719757080076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,8,128,1,float16,fp8,0,0.47708640098571775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,8,128,1,fp8,fp8,0,0.476855993270874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,48,128,1,float16,float16,0,0.41208958625793457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,48,128,1,float16,fp8,0,0.38645119667053224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,48,128,1,fp8,fp8,0,0.38634400367736815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,1,128,1,float16,float16,0,0.23664319515228271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,1,128,1,float16,fp8,0,0.2241584062576294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,1,128,1,fp8,fp8,0,0.22157280445098876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,2,128,1,float16,float16,0,0.2394416093826294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,2,128,1,float16,fp8,0,0.23337759971618652
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,2,128,1,fp8,fp8,0,0.23036160469055175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,4,128,1,float16,float16,0,0.24674880504608154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,4,128,1,float16,fp8,0,0.23652479648590088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,4,128,1,fp8,fp8,0,0.23688960075378418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,8,128,1,float16,float16,0,0.25307838916778563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,8,128,1,float16,fp8,0,0.24404480457305908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,8,128,1,fp8,fp8,0,0.2525360107421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,1,128,1,float16,fp8,0,6.28838882446289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,1,128,1,fp8,fp8,0,6.260214233398438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,1,128,1,float16,float16,0,6.950217437744141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,2,128,1,float16,fp8,0,6.395521545410157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,2,128,1,fp8,fp8,0,6.4611663818359375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,2,128,1,float16,float16,0,7.140404510498047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,4,128,1,float16,fp8,0,6.710655975341797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,4,128,1,float16,float16,0,7.516165161132813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,4,128,1,fp8,fp8,0,6.666187286376953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,8,128,1,float16,fp8,0,7.215315246582032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,48,128,1,float16,float16,0,6.924147033691407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,48,128,1,float16,fp8,0,6.4982353210449215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,8,128,1,fp8,fp8,0,7.240532684326172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,8,128,1,float16,float16,0,8.09100799560547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,48,128,1,fp8,fp8,0,6.472510528564453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,1,128,1,float16,fp8,0,3.1215936660766603
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,1,128,1,float16,float16,0,3.3760032653808594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,1,128,1,fp8,fp8,0,3.1041423797607424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,2,128,1,float16,fp8,0,3.2075359344482424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,2,128,1,fp8,fp8,0,3.1931840896606447
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,2,128,1,float16,float16,0,3.700112152099609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,4,128,1,float16,float16,0,3.7358497619628905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,4,128,1,float16,fp8,0,3.3538192749023437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,4,128,1,fp8,fp8,0,3.3414478302001953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,1,128,1,float16,float16,0,1.649038314819336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,8,128,1,float16,float16,0,4.074123382568359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,8,128,1,float16,fp8,0,3.6055633544921877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,48,128,1,float16,float16,0,3.420804977416992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,8,128,1,fp8,fp8,0,3.6406272888183593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,48,128,1,float16,fp8,0,3.361857604980469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,1,128,1,float16,fp8,0,1.5736672401428222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,48,128,1,fp8,fp8,0,3.282428741455078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,1,128,1,fp8,fp8,0,1.6177391052246093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,2,128,1,float16,float16,0,1.7333168029785155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,2,128,1,float16,fp8,0,1.5984031677246093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,2,128,1,fp8,fp8,0,1.636849594116211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,4,128,1,float16,float16,0,1.7617263793945312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,4,128,1,float16,fp8,0,1.687295913696289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,4,128,1,fp8,fp8,0,1.68164005279541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,8,128,1,float16,float16,0,1.9383232116699218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,8,128,1,float16,fp8,0,1.8702127456665039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,1,128,1,float16,float16,0,0.8388352394104004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,1,128,1,float16,fp8,0,0.7990767955780029
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,8,128,1,fp8,fp8,0,1.8766975402832031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,48,128,1,float16,float16,0,1.6975536346435547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,48,128,1,float16,fp8,0,1.6690719604492188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,1,128,1,fp8,fp8,0,0.8112895965576172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,48,128,1,fp8,fp8,0,1.6802656173706054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,2,128,1,float16,float16,0,0.86702880859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,2,128,1,float16,fp8,0,0.8303055763244629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,2,128,1,fp8,fp8,0,0.819001579284668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,4,128,1,float16,float16,0,0.9165727615356445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,4,128,1,float16,fp8,0,0.8481583595275879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,4,128,1,fp8,fp8,0,0.8694479942321778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,8,128,1,float16,float16,0,0.9751759529113769
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,8,128,1,float16,fp8,0,0.9535856246948242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,48,128,1,float16,float16,0,0.8742671966552734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,8,128,1,fp8,fp8,0,0.9459280014038086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,1,128,1,float16,float16,0,0.4253568172454834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,48,128,1,float16,fp8,0,0.8300623893737793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,1,128,1,float16,fp8,0,0.42229599952697755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,48,128,1,fp8,fp8,0,0.849118423461914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,1,128,1,fp8,fp8,0,0.433025598526001
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,2,128,1,float16,float16,0,0.4484111785888672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,2,128,1,float16,fp8,0,0.4330592155456543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,2,128,1,fp8,fp8,0,0.4322080135345459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,4,128,1,float16,float16,0,0.45246400833129885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,4,128,1,float16,fp8,0,0.44887681007385255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,4,128,1,fp8,fp8,0,0.44342880249023436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,8,128,1,float16,float16,0,0.5057631969451905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,8,128,1,float16,fp8,0,0.4873936176300049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,8,128,1,fp8,fp8,0,0.4880191802978516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,48,128,1,float16,float16,0,0.4583631992340088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,48,128,1,float16,fp8,0,0.4321904182434082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,1,128,1,float16,float16,0,0.23089120388031006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,48,128,1,fp8,fp8,0,0.4252448081970215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,1,128,1,float16,fp8,0,0.21988799571990966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,1,128,1,fp8,fp8,0,0.22262399196624755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,2,128,1,float16,float16,0,0.2369839906692505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,2,128,1,float16,fp8,0,0.23011679649353028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,2,128,1,fp8,fp8,0,0.22884480953216552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,4,128,1,float16,float16,0,0.24188001155853273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,4,128,1,float16,fp8,0,0.2364703893661499
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,4,128,1,fp8,fp8,0,0.2392496109008789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,8,128,1,float16,float16,0,0.26181280612945557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,8,128,1,float16,fp8,0,0.25642240047454834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,8,128,1,fp8,fp8,0,0.25862081050872804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,48,128,1,float16,float16,0,0.24274399280548095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,48,128,1,float16,fp8,0,0.230784010887146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,48,128,1,fp8,fp8,0,0.23184640407562257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,1,128,1,float16,float16,0,0.12939200401306153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,1,128,1,float16,fp8,0,0.12382719516754151
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,1,128,1,fp8,fp8,0,0.11974400281906128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,2,128,1,float16,float16,0,0.13042880296707154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,2,128,1,float16,fp8,0,0.12434879541397095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,2,128,1,fp8,fp8,0,0.1258239984512329
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,4,128,1,float16,float16,0,0.13549599647521973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,4,128,1,float16,fp8,0,0.13235039710998536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,4,128,1,fp8,fp8,0,0.13151040077209472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,8,128,1,float16,float16,0,0.14236960411071778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,8,128,1,float16,fp8,0,0.1380720019340515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,8,128,1,fp8,fp8,0,0.1403712034225464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,1,128,1,float16,fp8,0,3.9274112701416017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,1,128,1,float16,float16,0,4.183705520629883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,1,128,1,fp8,fp8,0,3.886500930786133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,2,128,1,float16,fp8,0,3.9966175079345705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,2,128,1,float16,float16,0,4.238808059692383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,2,128,1,fp8,fp8,0,4.024470520019531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,4,128,1,float16,float16,0,4.562555313110352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,4,128,1,float16,fp8,0,4.229344177246094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,4,128,1,fp8,fp8,0,4.195708847045898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,8,128,1,float16,float16,0,5.052238464355469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,8,128,1,float16,fp8,0,4.623116683959961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,8,128,1,fp8,fp8,0,4.648543930053711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,48,128,1,float16,float16,0,4.678721618652344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,48,128,1,fp8,fp8,0,4.452008056640625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,48,128,1,float16,fp8,0,4.505697631835938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,1,128,1,float16,float16,0,2.0613311767578124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,1,128,1,fp8,fp8,0,1.9418336868286132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,1,128,1,float16,fp8,0,2.026375961303711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,2,128,1,float16,float16,0,2.1065887451171874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,2,128,1,float16,fp8,0,2.038796806335449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,2,128,1,fp8,fp8,0,2.0010095596313477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,4,128,1,float16,float16,0,2.230134391784668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,4,128,1,float16,fp8,0,2.240795135498047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,4,128,1,fp8,fp8,0,2.107377624511719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,1,128,1,float16,float16,0,1.0549280166625976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,8,128,1,float16,float16,0,2.474361610412598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,8,128,1,float16,fp8,0,2.4115392684936525
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,48,128,1,float16,float16,0,2.3690319061279297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,8,128,1,fp8,fp8,0,2.3727344512939452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,1,128,1,float16,fp8,0,0.9835295677185059
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,48,128,1,float16,fp8,0,2.27380313873291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,48,128,1,fp8,fp8,0,2.3389631271362306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,1,128,1,fp8,fp8,0,0.9872207641601562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,2,128,1,float16,fp8,0,1.038862419128418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,2,128,1,float16,float16,0,1.0642720222473145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,2,128,1,fp8,fp8,0,1.028715229034424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,4,128,1,float16,float16,0,1.1016832351684571
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,4,128,1,float16,fp8,0,1.0826687812805176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,4,128,1,fp8,fp8,0,1.0744319915771485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,8,128,1,float16,float16,0,1.2325440406799317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,8,128,1,float16,fp8,0,1.1792048454284667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,8,128,1,fp8,fp8,0,1.18985595703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,1,128,1,float16,float16,0,0.5313983917236328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,48,128,1,float16,float16,0,1.206886386871338
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,1,128,1,float16,fp8,0,0.5075935840606689
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,48,128,1,float16,fp8,0,1.161575984954834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,48,128,1,fp8,fp8,0,1.1381711959838867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,1,128,1,fp8,fp8,0,0.5132656097412109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,2,128,1,float16,float16,0,0.5477935791015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,2,128,1,float16,fp8,0,0.5205344200134278
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,2,128,1,fp8,fp8,0,0.5295455932617188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,4,128,1,float16,float16,0,0.5762127876281739
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,4,128,1,float16,fp8,0,0.5477424144744873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,4,128,1,fp8,fp8,0,0.5546688079833985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,8,128,1,float16,float16,0,0.6308095932006836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,8,128,1,float16,fp8,0,0.6034463882446289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,8,128,1,fp8,fp8,0,0.6130080223083496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,1,128,1,float16,fp8,0,0.26900639533996584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,48,128,1,float16,float16,0,0.6036431789398193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,48,128,1,float16,fp8,0,0.5972064018249512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,1,128,1,float16,float16,0,0.27103519439697266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,48,128,1,fp8,fp8,0,0.5968927860260009
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,4,128,1,float16,float16,0,0.293556809425354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,1,128,1,fp8,fp8,0,0.2684959888458252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,2,128,1,float16,float16,0,0.2823744058609009
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,2,128,1,float16,fp8,0,0.2790191888809204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,2,128,1,fp8,fp8,0,0.2766207933425903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,4,128,1,float16,fp8,0,0.2924655914306641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,4,128,1,fp8,fp8,0,0.2867856025695801
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,8,128,1,fp8,fp8,0,0.31812798976898193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,8,128,1,float16,fp8,0,0.3141024112701416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,8,128,1,float16,float16,0,0.32914080619812014
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,48,128,1,float16,float16,0,0.32240478992462157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,48,128,1,float16,fp8,0,0.3121200084686279
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,48,128,1,fp8,fp8,0,0.31057279109954833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,1,128,1,float16,float16,0,0.15126240253448486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,1,128,1,float16,fp8,0,0.14652320146560668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,1,128,1,fp8,fp8,0,0.14953279495239258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,2,128,1,float16,float16,0,0.15508320331573486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,8,128,1,float16,float16,0,0.17432160377502443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,2,128,1,float16,fp8,0,0.15139360427856446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,2,128,1,fp8,fp8,0,0.15085439682006835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,4,128,1,float16,float16,0,0.16180959939956666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,4,128,1,float16,fp8,0,0.15707199573516845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,4,128,1,fp8,fp8,0,0.15641759634017943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,8,128,1,float16,fp8,0,0.17164160013198854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,8,128,1,fp8,fp8,0,0.1714192032814026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,48,128,1,float16,float16,0,0.17175840139389037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,48,128,1,float16,fp8,0,0.16759040355682372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,2,128,1,float16,fp8,0,0.08261600136756897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,48,128,1,fp8,fp8,0,0.16818079948425294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,1,128,1,float16,float16,0,0.08694080114364625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,1,128,1,float16,fp8,0,0.08168320059776306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,1,128,1,fp8,fp8,0,0.08036320209503174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,2,128,1,float16,float16,0,0.0879151999950409
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,2,128,1,fp8,fp8,0,0.08045920133590698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,4,128,1,float16,float16,0,0.09104639887809754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,4,128,1,float16,fp8,0,0.0856719970703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,4,128,1,fp8,fp8,0,0.08470879793167115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,8,128,1,float16,float16,0,0.09868800044059753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,8,128,1,float16,fp8,0,0.09356319904327393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,8,128,1,fp8,fp8,0,0.09467039704322815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,1,128,1,float16,float16,0,4.2929023742675785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,1,128,1,float16,fp8,0,4.0994926452636715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,1,128,1,fp8,fp8,0,4.107515335083008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,2,128,1,float16,fp8,0,4.248358535766601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,2,128,1,float16,float16,0,4.462817764282226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,2,128,1,fp8,fp8,0,4.260124969482422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,4,128,1,float16,float16,0,4.758868789672851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,4,128,1,float16,fp8,0,4.542307281494141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,4,128,1,fp8,fp8,0,4.5171455383300785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,8,128,1,float16,float16,0,5.329412841796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,8,128,1,float16,fp8,0,5.109337615966797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,8,128,1,fp8,fp8,0,5.096550369262696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,48,128,1,float16,float16,0,5.6659294128417965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,48,128,1,float16,fp8,0,5.413116836547852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,1,128,1,float16,float16,0,2.1063199996948243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,48,128,1,fp8,fp8,0,5.388734436035156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,1,128,1,float16,fp8,0,2.0547136306762694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,1,128,1,fp8,fp8,0,2.040007972717285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,2,128,1,float16,float16,0,2.184662437438965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,2,128,1,float16,fp8,0,2.1569679260253904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,2,128,1,fp8,fp8,0,2.13787841796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,4,128,1,float16,float16,0,2.360456085205078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,4,128,1,float16,fp8,0,2.275129508972168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,4,128,1,fp8,fp8,0,2.2610687255859374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,8,128,1,float16,float16,0,2.6573776245117187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,1,128,1,float16,float16,0,1.0560367584228516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,1,128,1,float16,fp8,0,1.029691219329834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,8,128,1,float16,fp8,0,2.5439599990844726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,8,128,1,fp8,fp8,0,2.6516624450683595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,1,128,1,fp8,fp8,0,1.084547233581543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,48,128,1,float16,float16,0,2.8284400939941405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,48,128,1,float16,fp8,0,2.7306543350219727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,2,128,1,float16,float16,0,1.1191776275634766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,2,128,1,float16,fp8,0,1.066158390045166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,48,128,1,fp8,fp8,0,2.727124786376953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,2,128,1,fp8,fp8,0,1.0667743682861328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,4,128,1,float16,float16,0,1.1970191955566407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,4,128,1,float16,fp8,0,1.1746560096740724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,4,128,1,fp8,fp8,0,1.1483792304992675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,8,128,1,float16,float16,0,1.3290143966674806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,8,128,1,float16,fp8,0,1.3225312232971191
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,8,128,1,fp8,fp8,0,1.2814000129699707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,1,128,1,float16,float16,0,0.5541440010070801
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,1,128,1,float16,fp8,0,0.5381760120391845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,48,128,1,float16,fp8,0,1.3605775833129883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,48,128,1,float16,float16,0,1.4227935791015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,48,128,1,fp8,fp8,0,1.3766176223754882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,1,128,1,fp8,fp8,0,0.5322527885437012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,2,128,1,float16,float16,0,0.5766416072845459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,2,128,1,float16,fp8,0,0.5452144145965576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,2,128,1,fp8,fp8,0,0.5505263805389404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,4,128,1,float16,float16,0,0.6079311847686768
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,4,128,1,float16,fp8,0,0.5798992156982422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,4,128,1,fp8,fp8,0,0.5989391803741455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,8,128,1,float16,float16,0,0.6741759777069092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,8,128,1,float16,fp8,0,0.6582623958587647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,8,128,1,fp8,fp8,0,0.6575439929962158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,48,128,1,float16,float16,0,0.719920015335083
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,1,128,1,float16,float16,0,0.2814496040344238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,48,128,1,float16,fp8,0,0.7022079944610595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,1,128,1,float16,fp8,0,0.2852128028869629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,48,128,1,fp8,fp8,0,0.7114672183990478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,1,128,1,fp8,fp8,0,0.27926719188690186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,2,128,1,float16,float16,0,0.29291999340057373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,2,128,1,float16,fp8,0,0.28696000576019287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,2,128,1,fp8,fp8,0,0.28450560569763184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,4,128,1,float16,float16,0,0.31547999382019043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,4,128,1,float16,fp8,0,0.30533599853515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,4,128,1,fp8,fp8,0,0.3068928003311157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,8,128,1,float16,float16,0,0.35524959564208985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,8,128,1,float16,fp8,0,0.34078240394592285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,8,128,1,fp8,fp8,0,0.3408512115478516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,48,128,1,float16,float16,0,0.37592320442199706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,48,128,1,float16,fp8,0,0.3591088056564331
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,2,128,1,float16,fp8,0,0.1565999984741211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,1,128,1,float16,float16,0,0.15203200578689574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,48,128,1,fp8,fp8,0,0.3677119970321655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,1,128,1,float16,fp8,0,0.15204639434814454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,1,128,1,fp8,fp8,0,0.15325599908828735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,2,128,1,float16,float16,0,0.15895040035247804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,2,128,1,fp8,fp8,0,0.1559376001358032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,4,128,1,float16,float16,0,0.1705183982849121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,4,128,1,float16,fp8,0,0.16452640295028687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,4,128,1,fp8,fp8,0,0.1642351984977722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,8,128,1,float16,float16,0,0.18853440284729003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,8,128,1,float16,fp8,0,0.1841488003730774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,8,128,1,fp8,fp8,0,0.1854591965675354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,48,128,1,float16,float16,0,0.2051759958267212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,48,128,1,float16,fp8,0,0.1925696015357971
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,2,128,1,float16,fp8,0,0.08927040100097657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,48,128,1,fp8,fp8,0,0.1966591954231262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,1,128,1,float16,float16,0,0.08882079720497131
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,1,128,1,float16,fp8,0,0.08536480069160461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,4,128,1,fp8,fp8,0,0.09453439712524414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,1,128,1,fp8,fp8,0,0.08880640268325805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,2,128,1,float16,float16,0,0.09020959734916686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,2,128,1,fp8,fp8,0,0.09056479930877685
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,4,128,1,float16,float16,0,0.09454240202903748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,4,128,1,float16,fp8,0,0.09494879841804504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,8,128,1,float16,float16,0,0.1031999945640564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,8,128,1,float16,fp8,0,0.10267200469970703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,8,128,1,fp8,fp8,0,0.10353440046310425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,48,128,1,float16,float16,0,0.11475679874420167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,48,128,1,float16,fp8,0,0.10952639579772949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,48,128,1,fp8,fp8,0,0.11088160276412964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,1,128,1,float16,float16,0,0.0510640025138855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,1,128,1,float16,fp8,0,0.04973120093345642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,1,128,1,fp8,fp8,0,0.05066559910774231
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,2,128,1,float16,float16,0,0.05027040243148804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,2,128,1,float16,fp8,0,0.050169599056243894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,2,128,1,fp8,fp8,0,0.04943839907646179
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,4,128,1,float16,float16,0,0.05262399911880493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,4,128,1,float16,fp8,0,0.05071520209312439
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,4,128,1,fp8,fp8,0,0.05068960189819336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,8,128,1,float16,float16,0,0.059755200147628786
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,8,128,1,float16,fp8,0,0.05598719716072083
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,8,128,1,fp8,fp8,0,0.05580639839172363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,1,128,1,float16,float16,0,2.954089546203613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,1,128,1,float16,fp8,0,2.984391975402832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,1,128,1,fp8,fp8,0,2.9815759658813477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,2,128,1,float16,float16,0,3.08895206451416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,2,128,1,float16,fp8,0,3.1585983276367187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,2,128,1,fp8,fp8,0,3.13175048828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,4,128,1,float16,float16,0,3.4079071044921876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,4,128,1,float16,fp8,0,3.40849609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,4,128,1,fp8,fp8,0,3.404622268676758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,8,128,1,float16,float16,0,3.991388702392578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,8,128,1,float16,fp8,0,4.031679916381836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,8,128,1,fp8,fp8,0,4.016265487670898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,1,128,1,float16,float16,0,1.464851188659668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,48,128,1,float16,float16,0,5.01402702331543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,1,128,1,float16,fp8,0,1.5105104446411133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,48,128,1,float16,fp8,0,4.91926383972168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,1,128,1,fp8,fp8,0,1.4811264038085938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,2,128,1,float16,float16,0,1.5457263946533204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,2,128,1,float16,fp8,0,1.5654303550720214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,2,128,1,fp8,fp8,0,1.5652864456176758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,4,128,1,float16,float16,0,1.7021728515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,4,128,1,float16,fp8,0,1.7224720001220704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,4,128,1,fp8,fp8,0,1.7169343948364257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,48,128,1,fp8,fp8,0,4.891584014892578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,8,128,1,float16,float16,0,2.002039909362793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,8,128,1,float16,fp8,0,2.00230712890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,8,128,1,fp8,fp8,0,2.0093711853027343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,1,128,1,float16,float16,0,0.7462607860565186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,1,128,1,float16,fp8,0,0.7614304065704346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,1,128,1,fp8,fp8,0,0.7584527969360352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,2,128,1,float16,float16,0,0.7806719779968262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,48,128,1,float16,float16,0,2.5107664108276366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,2,128,1,float16,fp8,0,0.7951903820037842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,2,128,1,fp8,fp8,0,0.7986735820770263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,48,128,1,float16,fp8,0,2.4817760467529295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,48,128,1,fp8,fp8,0,2.4677839279174805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,4,128,1,float16,float16,0,0.8624640464782715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,4,128,1,float16,fp8,0,0.8652400016784668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,4,128,1,fp8,fp8,0,0.861030387878418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,8,128,1,float16,float16,0,1.0165663719177247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,8,128,1,float16,fp8,0,1.014355182647705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,1,128,1,float16,float16,0,0.38259360790252683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,1,128,1,float16,fp8,0,0.3889039993286133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,8,128,1,fp8,fp8,0,1.0157247543334962
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,1,128,1,fp8,fp8,0,0.39126560688018797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,48,128,1,float16,float16,0,1.2968544006347655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,2,128,1,float16,float16,0,0.4042384147644043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,48,128,1,float16,fp8,0,1.2449392318725585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,2,128,1,float16,fp8,0,0.40744638442993164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,48,128,1,fp8,fp8,0,1.2391231536865235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,2,128,1,fp8,fp8,0,0.40786399841308596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,4,128,1,float16,float16,0,0.4498960018157959
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,4,128,1,float16,fp8,0,0.4449728012084961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,8,128,1,fp8,fp8,0,0.5149775981903076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,4,128,1,fp8,fp8,0,0.4520991802215576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,8,128,1,float16,float16,0,0.5201903820037842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,8,128,1,float16,fp8,0,0.5216447830200195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,48,128,1,float16,float16,0,0.6585279941558838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,1,128,1,float16,float16,0,0.2051408052444458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,48,128,1,float16,fp8,0,0.6310832023620605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,1,128,1,float16,fp8,0,0.2073296070098877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,48,128,1,fp8,fp8,0,0.6458608150482178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,1,128,1,fp8,fp8,0,0.21079521179199218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,2,128,1,float16,float16,0,0.21347360610961913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,2,128,1,float16,fp8,0,0.21844000816345216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,2,128,1,fp8,fp8,0,0.21988000869750976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,4,128,1,float16,float16,0,0.23322720527648927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,4,128,1,float16,fp8,0,0.23406720161437988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,4,128,1,fp8,fp8,0,0.23315200805664063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,8,128,1,float16,float16,0,0.27587840557098386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,1,128,1,float16,float16,0,0.11399040222167969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,8,128,1,float16,fp8,0,0.27451519966125487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,8,128,1,fp8,fp8,0,0.2759648084640503
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,48,128,1,float16,float16,0,0.3398832082748413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,48,128,1,float16,fp8,0,0.3346240043640137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,48,128,1,fp8,fp8,0,0.32830560207366943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,1,128,1,float16,fp8,0,0.11579359769821167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,1,128,1,fp8,fp8,0,0.1180575966835022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,4,128,1,fp8,fp8,0,0.13182239532470702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,2,128,1,float16,float16,0,0.12001440525054932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,2,128,1,float16,fp8,0,0.12307360172271728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,2,128,1,fp8,fp8,0,0.12428319454193115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,4,128,1,float16,float16,0,0.12887040376663209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,4,128,1,float16,fp8,0,0.13118079900741578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,8,128,1,float16,float16,0,0.1482591986656189
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,1,128,1,float16,float16,0,0.0706816017627716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,8,128,1,float16,fp8,0,0.15112160444259642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,8,128,1,fp8,fp8,0,0.1511008024215698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,48,128,1,float16,float16,0,0.18551520109176636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,48,128,1,float16,fp8,0,0.17848800420761107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,48,128,1,fp8,fp8,0,0.17933759689331055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,1,128,1,float16,fp8,0,0.07059199810028076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,1,128,1,fp8,fp8,0,0.07060480117797852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,2,128,1,float16,float16,0,0.0719215989112854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,2,128,1,float16,fp8,0,0.07283200025558471
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,2,128,1,fp8,fp8,0,0.07232159972190857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,4,128,1,float16,float16,0,0.07747039794921876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,4,128,1,float16,fp8,0,0.0786512017250061
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,4,128,1,fp8,fp8,0,0.07823200225830078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,8,128,1,float16,float16,0,0.08551520109176636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,8,128,1,float16,fp8,0,0.08636479973793029
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,8,128,1,fp8,fp8,0,0.08729280233383178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,48,128,1,float16,float16,0,0.10440640449523926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,48,128,1,float16,fp8,0,0.10057120323181153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,48,128,1,fp8,fp8,0,0.10128959417343139
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,1,128,1,float16,float16,0,0.04261919856071472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,1,128,1,float16,fp8,0,0.04264959990978241
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,1,128,1,fp8,fp8,0,0.04339199960231781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,2,128,1,float16,float16,0,0.04354400038719177
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,2,128,1,float16,fp8,0,0.043699198961257936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,2,128,1,fp8,fp8,0,0.04337919950485229
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,4,128,1,float16,float16,0,0.04472000002861023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,4,128,1,float16,fp8,0,0.0443664014339447
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,4,128,1,fp8,fp8,0,0.044142401218414305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,8,128,1,float16,float16,0,0.051313602924346925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,8,128,1,float16,fp8,0,0.05037599802017212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,8,128,1,fp8,fp8,0,0.05150880217552185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,48,128,1,float16,float16,0,0.06081600189208984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,48,128,1,float16,fp8,0,0.05529919862747192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,48,128,1,fp8,fp8,0,0.05556319952011109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,1,128,1,float16,float16,0,0.031692799925804135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,1,128,1,float16,fp8,0,0.03165920078754425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,1,128,1,fp8,fp8,0,0.03182399868965149
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,2,128,1,float16,float16,0,0.0312608003616333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,2,128,1,float16,fp8,0,0.03155519962310791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,2,128,1,fp8,fp8,0,0.031113600730895995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,4,128,1,float16,float16,0,0.03197599947452545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,4,128,1,float16,fp8,0,0.03162400126457214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,4,128,1,fp8,fp8,0,0.031625598669052124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,8,128,1,float16,float16,0,0.03260799944400787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,8,128,1,float16,fp8,0,0.03444480001926422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,8,128,1,fp8,fp8,0,0.03436479866504669
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,1,128,1,float16,float16,0,1.1847968101501465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,1,128,1,float16,fp8,0,1.2401599884033203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,1,128,1,fp8,fp8,0,1.2489423751831055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,2,128,1,float16,float16,0,1.2633055686950683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,2,128,1,float16,fp8,0,1.3159536361694335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,2,128,1,fp8,fp8,0,1.3169631958007812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,4,128,1,float16,float16,0,1.419985580444336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,4,128,1,float16,fp8,0,1.4629856109619142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,4,128,1,fp8,fp8,0,1.4615792274475097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,8,128,1,float16,float16,0,1.7162063598632813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,1,128,1,float16,float16,0,0.6013919830322265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,8,128,1,float16,fp8,0,1.7671039581298829
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,1,128,1,float16,fp8,0,0.6326288223266602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,8,128,1,fp8,fp8,0,1.7549583435058593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,1,128,1,fp8,fp8,0,0.63230881690979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,2,128,1,float16,float16,0,0.6397136211395263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,48,128,1,float16,float16,0,2.436604881286621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,2,128,1,float16,fp8,0,0.6685344219207764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,48,128,1,float16,fp8,0,2.329944038391113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,2,128,1,fp8,fp8,0,0.6698927879333496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,48,128,1,fp8,fp8,0,2.344363212585449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,4,128,1,float16,float16,0,0.7159296035766601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,4,128,1,float16,fp8,0,0.7391456127166748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,4,128,1,fp8,fp8,0,0.739896011352539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,8,128,1,float16,float16,0,0.8711248397827148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,8,128,1,float16,fp8,0,0.885108757019043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,8,128,1,fp8,fp8,0,0.8883008003234864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,1,128,1,float16,float16,0,0.31113600730895996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,1,128,1,float16,fp8,0,0.32828478813171386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,48,128,1,float16,float16,0,1.221292781829834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,2,128,1,float16,float16,0,0.33191680908203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,1,128,1,fp8,fp8,0,0.32779679298400877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,48,128,1,float16,fp8,0,1.1752304077148437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,48,128,1,fp8,fp8,0,1.1658063888549806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,2,128,1,float16,fp8,0,0.3470799922943115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,2,128,1,fp8,fp8,0,0.34447360038757324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,4,128,1,float16,float16,0,0.3699615955352783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,4,128,1,float16,fp8,0,0.38072159290313723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,4,128,1,fp8,fp8,0,0.3805824041366577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,8,128,1,float16,float16,0,0.4458144187927246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,8,128,1,float16,fp8,0,0.45638718605041506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,8,128,1,fp8,fp8,0,0.45356478691101076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,1,128,1,float16,float16,0,0.16885440349578856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,48,128,1,float16,float16,0,0.6280416011810303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,48,128,1,float16,fp8,0,0.5912432193756103
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,1,128,1,float16,fp8,0,0.17669440507888795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,48,128,1,fp8,fp8,0,0.5955935955047608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,1,128,1,fp8,fp8,0,0.17691999673843384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,2,128,1,float16,float16,0,0.17816959619522094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,2,128,1,float16,fp8,0,0.18464959859848024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,2,128,1,fp8,fp8,0,0.18758080005645753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,4,128,1,float16,float16,0,0.19728319644927977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,4,128,1,float16,fp8,0,0.20219841003417968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,4,128,1,fp8,fp8,0,0.2023632049560547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,8,128,1,float16,float16,0,0.23451039791107178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,8,128,1,float16,fp8,0,0.23930718898773193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,8,128,1,fp8,fp8,0,0.24018239974975586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,48,128,1,float16,float16,0,0.3256256103515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,48,128,1,float16,fp8,0,0.31498239040374754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,1,128,1,float16,float16,0,0.09631360173225403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,48,128,1,fp8,fp8,0,0.31072800159454345
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,1,128,1,float16,fp8,0,0.09952960014343262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,1,128,1,fp8,fp8,0,0.09898399710655212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,2,128,1,float16,float16,0,0.1021183967590332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,2,128,1,float16,fp8,0,0.10487040281295776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,2,128,1,fp8,fp8,0,0.10673439502716064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,4,128,1,float16,float16,0,0.10979679822921753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,4,128,1,float16,fp8,0,0.11304639577865601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,4,128,1,fp8,fp8,0,0.11330879926681518
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,8,128,1,float16,float16,0,0.1293887972831726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,8,128,1,float16,fp8,0,0.13163360357284545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,8,128,1,fp8,fp8,0,0.13147200345993043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,48,128,1,float16,float16,0,0.17781440019607545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,48,128,1,float16,fp8,0,0.16775200366973878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,48,128,1,fp8,fp8,0,0.1679152011871338
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,1,128,1,float16,float16,0,0.06063359975814819
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,1,128,1,float16,fp8,0,0.060755199193954466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,1,128,1,fp8,fp8,0,0.061566400527954104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,2,128,1,float16,float16,0,0.06207200288772583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,2,128,1,float16,fp8,0,0.0629360020160675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,2,128,1,fp8,fp8,0,0.06213120222091675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,4,128,1,float16,float16,0,0.06754400134086609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,4,128,1,float16,fp8,0,0.0689631998538971
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,4,128,1,fp8,fp8,0,0.06839039921760559
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,8,128,1,float16,float16,0,0.07595840096473694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,8,128,1,float16,fp8,0,0.07691360116004944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,8,128,1,fp8,fp8,0,0.07816960215568543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,48,128,1,float16,float16,0,0.10000959634780884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,48,128,1,float16,fp8,0,0.09511680006980897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,48,128,1,fp8,fp8,0,0.09463840126991271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,1,128,1,float16,float16,0,0.03625119924545288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,1,128,1,float16,fp8,0,0.03798879981040955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,1,128,1,fp8,fp8,0,0.037827199697494505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,2,128,1,float16,fp8,0,0.038446399569511416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,2,128,1,float16,float16,0,0.03748640120029449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,2,128,1,fp8,fp8,0,0.03826079964637756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,4,128,1,float16,float16,0,0.038736000657081604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,4,128,1,float16,fp8,0,0.03895840048789978
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,4,128,1,fp8,fp8,0,0.03893280029296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,8,128,1,float16,float16,0,0.04550240039825439
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,8,128,1,float16,fp8,0,0.04387199878692627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,8,128,1,fp8,fp8,0,0.04465599954128265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,48,128,1,float16,float16,0,0.05722079873085022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,48,128,1,float16,fp8,0,0.05133600234985351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,48,128,1,fp8,fp8,0,0.05101439952850342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,1,128,1,float16,float16,0,0.02953439950942993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,1,128,1,float16,fp8,0,0.029495999217033386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,1,128,1,fp8,fp8,0,0.029315200448036195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,2,128,1,float16,float16,0,0.0295199990272522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,2,128,1,float16,fp8,0,0.02934719920158386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,2,128,1,fp8,fp8,0,0.02935839891433716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,4,128,1,float16,float16,0,0.02961280047893524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,4,128,1,float16,fp8,0,0.029471999406814574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,4,128,1,fp8,fp8,0,0.029763200879096986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,8,128,1,float16,float16,0,0.03075360059738159
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,8,128,1,float16,fp8,0,0.032041600346565245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,8,128,1,fp8,fp8,0,0.031974399089813234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,48,128,1,float16,float16,0,0.03333120048046112
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,48,128,1,float16,fp8,0,0.034046399593353274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,48,128,1,fp8,fp8,0,0.034020799398422244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,1,128,1,float16,float16,0,0.02401279956102371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,1,128,1,float16,fp8,0,0.025179201364517213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,1,128,1,fp8,fp8,0,0.025128000974655153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,2,128,1,float16,float16,0,0.0239424005150795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,2,128,1,float16,fp8,0,0.02476000040769577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,2,128,1,fp8,fp8,0,0.024916799366474153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,4,128,1,float16,float16,0,0.024476799368858337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,4,128,1,float16,fp8,0,0.024697600305080412
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,4,128,1,fp8,fp8,0,0.025080001354217528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,8,128,1,float16,float16,0,0.024486400187015533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,8,128,1,float16,fp8,0,0.025092801451683043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,8,128,1,fp8,fp8,0,0.0251008003950119
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,1,128,1,float16,float16,0,0.5764671802520752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,1,128,1,float16,fp8,0,0.6186768054962158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,1,128,1,fp8,fp8,0,0.6200975894927978
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,2,128,1,float16,float16,0,0.6114287853240967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,2,128,1,float16,fp8,0,0.6646480083465576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,2,128,1,fp8,fp8,0,0.6585887908935547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,4,128,1,float16,float16,0,0.6865744113922119
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,4,128,1,float16,fp8,0,0.7425487995147705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,8,128,1,float16,float16,0,0.8369503974914551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,4,128,1,fp8,fp8,0,0.7448112010955811
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,8,128,1,float16,fp8,0,0.8970239639282227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,8,128,1,fp8,fp8,0,0.8965007781982421
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,1,128,1,float16,float16,0,0.2995728015899658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,1,128,1,float16,fp8,0,0.31952800750732424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,1,128,1,fp8,fp8,0,0.3217983961105347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,48,128,1,float16,float16,0,1.2339808464050293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,2,128,1,float16,float16,0,0.3179296016693115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,48,128,1,float16,fp8,0,1.199232006072998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,48,128,1,fp8,fp8,0,1.1948255538940429
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,2,128,1,float16,fp8,0,0.3426736116409302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,2,128,1,fp8,fp8,0,0.3427455902099609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,4,128,1,float16,float16,0,0.35381920337677003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,4,128,1,float16,fp8,0,0.38438720703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,4,128,1,fp8,fp8,0,0.3838399887084961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,8,128,1,float16,float16,0,0.42678399085998536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,8,128,1,float16,fp8,0,0.4604320049285889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,8,128,1,fp8,fp8,0,0.45882081985473633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,48,128,1,float16,float16,0,0.629960012435913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,48,128,1,float16,fp8,0,0.6069664001464844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,2,128,1,float16,float16,0,0.17111200094223022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,1,128,1,float16,float16,0,0.1615504026412964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,2,128,1,float16,fp8,0,0.18151999711990358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,48,128,1,fp8,fp8,0,0.6075088024139405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,1,128,1,float16,fp8,0,0.17345759868621827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,1,128,1,fp8,fp8,0,0.17212480306625366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,8,128,1,float16,float16,0,0.2255295991897583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,2,128,1,fp8,fp8,0,0.18198879957199096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,4,128,1,float16,float16,0,0.18833919763565063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,4,128,1,float16,fp8,0,0.203108811378479
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,4,128,1,fp8,fp8,0,0.20267999172210693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,8,128,1,float16,fp8,0,0.24170238971710206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,8,128,1,fp8,fp8,0,0.24166719913482665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,48,128,1,float16,float16,0,0.3273200035095215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,48,128,1,float16,fp8,0,0.31284160614013673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,48,128,1,fp8,fp8,0,0.3134912014007568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,1,128,1,float16,float16,0,0.09020959734916686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,1,128,1,float16,fp8,0,0.09745919704437256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,1,128,1,fp8,fp8,0,0.09785119891166687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,2,128,1,float16,float16,0,0.09785599708557129
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,2,128,1,float16,fp8,0,0.10393760204315186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,8,128,1,float16,float16,0,0.12506879568099977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,2,128,1,fp8,fp8,0,0.10377600193023681
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,4,128,1,float16,float16,0,0.10514240264892578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,4,128,1,float16,fp8,0,0.11167360544204712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,4,128,1,fp8,fp8,0,0.11209440231323242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,48,128,1,fp8,fp8,0,0.16599520444869995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,8,128,1,float16,fp8,0,0.13027679920196533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,8,128,1,fp8,fp8,0,0.13048800230026245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,48,128,1,float16,float16,0,0.17687360048294068
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,48,128,1,float16,fp8,0,0.16595200300216675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,1,128,1,float16,float16,0,0.05851200222969055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,1,128,1,float16,fp8,0,0.0590224027633667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,1,128,1,fp8,fp8,0,0.05799840092658996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,2,128,1,float16,float16,0,0.059849601984024045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,2,128,1,float16,fp8,0,0.05996800065040588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,8,128,1,fp8,fp8,0,0.07505599856376648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,2,128,1,fp8,fp8,0,0.059575998783111574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,4,128,1,float16,float16,0,0.06570720076560974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,4,128,1,float16,fp8,0,0.06643360257148742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,4,128,1,fp8,fp8,0,0.06605439782142639
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,8,128,1,float16,float16,0,0.07418079972267151
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,8,128,1,float16,fp8,0,0.07454079985618592
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,48,128,1,float16,float16,0,0.097980797290802
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,48,128,1,float16,fp8,0,0.09063519835472107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,48,128,1,fp8,fp8,0,0.09021919965744019
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,1,128,1,float16,float16,0,0.03348479866981506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,1,128,1,float16,fp8,0,0.034944000840187076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,1,128,1,fp8,fp8,0,0.035227200388908385
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,2,128,1,float16,float16,0,0.03434880077838898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,2,128,1,float16,fp8,0,0.035257598757743834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,2,128,1,fp8,fp8,0,0.03532480001449585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,4,128,1,float16,float16,0,0.03528479933738708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,4,128,1,float16,fp8,0,0.03602400124073028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,4,128,1,fp8,fp8,0,0.036190399527549745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,8,128,1,float16,float16,0,0.04348320066928864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,8,128,1,float16,fp8,0,0.04078400135040283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,8,128,1,fp8,fp8,0,0.040766400098800656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,48,128,1,float16,float16,0,0.0561631977558136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,48,128,1,float16,fp8,0,0.049395200610160825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,48,128,1,fp8,fp8,0,0.0492688000202179
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,1,128,1,float16,float16,0,0.02815999984741211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,1,128,1,float16,fp8,0,0.029372799396514892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,1,128,1,fp8,fp8,0,0.029462400078773498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,2,128,1,float16,float16,0,0.028207999467849732
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,2,128,1,float16,fp8,0,0.029473599791526795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,2,128,1,fp8,fp8,0,0.029128000140190125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,4,128,1,float16,float16,0,0.028531199693679808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,4,128,1,float16,fp8,0,0.029528000950813295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,4,128,1,fp8,fp8,0,0.02953920066356659
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,8,128,1,float16,float16,0,0.029339200258255003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,8,128,1,float16,fp8,0,0.030380800366401672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,8,128,1,fp8,fp8,0,0.030110400915145875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,48,128,1,float16,float16,0,0.03219360113143921
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,48,128,1,float16,fp8,0,0.03218719959259033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,48,128,1,fp8,fp8,0,0.03218719959259033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,1,128,1,float16,float16,0,0.022652800381183624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,1,128,1,float16,fp8,0,0.023161600530147552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,1,128,1,fp8,fp8,0,0.023665599524974823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,2,128,1,float16,float16,0,0.022256000339984892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,2,128,1,float16,fp8,0,0.023387199640274046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,2,128,1,fp8,fp8,0,0.023076799511909486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,4,128,1,float16,float16,0,0.022801600396633148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,4,128,1,float16,fp8,0,0.02316959947347641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,4,128,1,fp8,fp8,0,0.023489600419998168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,8,128,1,float16,float16,0,0.023025600612163542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,8,128,1,float16,fp8,0,0.023395200073719025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,8,128,1,fp8,fp8,0,0.02327360063791275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,48,128,1,float16,float16,0,0.024967999756336214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,48,128,1,float16,fp8,0,0.025654399394989015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,48,128,1,fp8,fp8,0,0.025430399179458617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,1,128,1,float16,float16,0,0.021499200165271758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,1,128,1,float16,fp8,0,0.022603200376033784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,1,128,1,fp8,fp8,0,0.022094400227069856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,2,128,1,float16,float16,0,0.021428799629211424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,2,128,1,float16,fp8,0,0.022174400091171265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,2,128,1,fp8,fp8,0,0.022380800545215608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,4,128,1,float16,float16,0,0.021660800278186797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,4,128,1,float16,fp8,0,0.022448000311851502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,4,128,1,fp8,fp8,0,0.022142399847507478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,8,128,1,float16,float16,0,0.021657599508762358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,8,128,1,float16,fp8,0,0.022468799352645875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,8,128,1,fp8,fp8,0,0.022782400250434875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,1,128,1,float16,float16,0,0.30252320766448976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,1,128,1,float16,fp8,0,0.3271536111831665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,1,128,1,fp8,fp8,0,0.32525599002838135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,2,128,1,float16,float16,0,0.32015841007232665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,2,128,1,float16,fp8,0,0.3493583917617798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,2,128,1,fp8,fp8,0,0.3465359926223755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,4,128,1,float16,float16,0,0.35611519813537595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,4,128,1,float16,fp8,0,0.385097599029541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,4,128,1,fp8,fp8,0,0.38700640201568604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,8,128,1,float16,float16,0,0.4341743946075439
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,8,128,1,float16,fp8,0,0.45836639404296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,8,128,1,fp8,fp8,0,0.4621488094329834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,48,128,1,float16,float16,0,0.6961328029632569
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,1,128,1,float16,float16,0,0.16406559944152832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,48,128,1,float16,fp8,0,0.7134064197540283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,1,128,1,float16,fp8,0,0.17432639598846436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,48,128,1,fp8,fp8,0,0.7148911952972412
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,1,128,1,fp8,fp8,0,0.1722607970237732
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,2,128,1,float16,float16,0,0.17202240228652954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,2,128,1,float16,fp8,0,0.18414560556411744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,2,128,1,fp8,fp8,0,0.18583359718322753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,4,128,1,float16,float16,0,0.19095040559768678
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,48,128,1,float16,float16,0,0.36182079315185545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,4,128,1,float16,fp8,0,0.2020927906036377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,4,128,1,fp8,fp8,0,0.2044528007507324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,8,128,1,float16,float16,0,0.22726240158081054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,8,128,1,float16,fp8,0,0.24208641052246094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,8,128,1,fp8,fp8,0,0.2423759937286377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,48,128,1,float16,fp8,0,0.36786398887634275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,1,128,1,float16,float16,0,0.0919920027256012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,48,128,1,fp8,fp8,0,0.3686800003051758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,1,128,1,float16,fp8,0,0.09908000230789185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,1,128,1,fp8,fp8,0,0.09893760085105896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,2,128,1,float16,float16,0,0.09649440050125122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,2,128,1,float16,fp8,0,0.10551520586013793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,2,128,1,fp8,fp8,0,0.10503040552139283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,4,128,1,float16,float16,0,0.10568959712982177
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,4,128,1,float16,fp8,0,0.1126736044883728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,4,128,1,fp8,fp8,0,0.11318399906158447
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,8,128,1,float16,float16,0,0.12505919933319093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,8,128,1,float16,fp8,0,0.13207679986953735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,8,128,1,fp8,fp8,0,0.13193440437316895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,48,128,1,float16,float16,0,0.19421759843826295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,48,128,1,float16,fp8,0,0.19370399713516234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,48,128,1,fp8,fp8,0,0.19353920221328735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,1,128,1,float16,float16,0,0.059721601009368894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,1,128,1,float16,fp8,0,0.058371198177337644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,1,128,1,fp8,fp8,0,0.05947520136833191
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,2,128,1,float16,float16,0,0.060008001327514646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,2,128,1,float16,fp8,0,0.05989760160446167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,8,128,1,float16,fp8,0,0.07582719922065735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,2,128,1,fp8,fp8,0,0.06004959940910339
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,4,128,1,float16,float16,0,0.06562079787254334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,4,128,1,float16,fp8,0,0.06698560118675231
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,4,128,1,fp8,fp8,0,0.0650543987751007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,8,128,1,float16,float16,0,0.07389600276947021
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,8,128,1,fp8,fp8,0,0.07492640018463134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,48,128,1,float16,float16,0,0.10815199613571166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,48,128,1,float16,fp8,0,0.10436480045318604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,48,128,1,fp8,fp8,0,0.10320800542831421
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,1,128,1,float16,float16,0,0.033022400736808774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,1,128,1,float16,fp8,0,0.034827199578285215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,1,128,1,fp8,fp8,0,0.03504480123519897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,2,128,1,float16,float16,0,0.033980798721313474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,2,128,1,float16,fp8,0,0.03518719971179962
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,2,128,1,fp8,fp8,0,0.03510879874229431
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,4,128,1,float16,float16,0,0.03559040129184723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,4,128,1,float16,fp8,0,0.03609119951725006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,4,128,1,fp8,fp8,0,0.03566080033779144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,8,128,1,float16,float16,0,0.04171839952468872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,8,128,1,float16,fp8,0,0.04158239960670471
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,8,128,1,fp8,fp8,0,0.04139359891414642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,48,128,1,float16,float16,0,0.05879359841346741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,48,128,1,float16,fp8,0,0.053547197580337526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,48,128,1,fp8,fp8,0,0.05400480031967163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,1,128,1,float16,float16,0,0.028281599283218384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,1,128,1,float16,fp8,0,0.0292959988117218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,1,128,1,fp8,fp8,0,0.029267200827598573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,2,128,1,float16,float16,0,0.028119999170303344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,2,128,1,float16,fp8,0,0.029420799016952513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,2,128,1,fp8,fp8,0,0.02935360074043274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,4,128,1,float16,float16,0,0.028433600068092348
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,4,128,1,float16,fp8,0,0.029835200309753417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,4,128,1,fp8,fp8,0,0.029531198740005492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,8,128,1,float16,float16,0,0.029097598791122437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,8,128,1,float16,fp8,0,0.03023360073566437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,8,128,1,fp8,fp8,0,0.03039200007915497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,48,128,1,float16,float16,0,0.03484959900379181
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,48,128,1,float16,fp8,0,0.03618240058422088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,48,128,1,fp8,fp8,0,0.03610239923000336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,1,128,1,float16,float16,0,0.02263360023498535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,1,128,1,float16,fp8,0,0.023080000281333925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,1,128,1,fp8,fp8,0,0.023099200427532197
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,2,128,1,float16,float16,0,0.022430400550365447
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,2,128,1,float16,fp8,0,0.023313599824905395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,2,128,1,fp8,fp8,0,0.023174400627613067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,4,128,1,float16,float16,0,0.02258560061454773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,4,128,1,float16,fp8,0,0.02358720004558563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,4,128,1,fp8,fp8,0,0.023051199316978455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,8,128,1,float16,float16,0,0.02274399995803833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,8,128,1,float16,fp8,0,0.023559999465942384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,8,128,1,fp8,fp8,0,0.02341119945049286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,48,128,1,float16,float16,0,0.0244719997048378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,48,128,1,float16,fp8,0,0.02494720071554184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,48,128,1,fp8,fp8,0,0.02529120147228241
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,1,128,1,float16,float16,0,0.021489599347114564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,1,128,1,float16,fp8,0,0.021984000504016877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,1,128,1,fp8,fp8,0,0.021988800168037413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,2,128,1,float16,float16,0,0.021561600267887115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,2,128,1,float16,fp8,0,0.022459200024604796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,2,128,1,fp8,fp8,0,0.022316800057888032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,4,128,1,float16,float16,0,0.021729600429534913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,4,128,1,float16,fp8,0,0.022308799624443054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,48,128,1,float16,float16,0,0.021855999529361726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,4,128,1,fp8,fp8,0,0.02221280038356781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,8,128,1,float16,float16,0,0.021524800360202788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,8,128,1,float16,fp8,0,0.022836799919605254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,8,128,1,fp8,fp8,0,0.022444799542427063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,48,128,1,float16,fp8,0,0.022463999688625336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,48,128,1,fp8,fp8,0,0.022526399791240694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,1,128,1,float16,float16,0,0.020815999805927278
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,1,128,1,float16,fp8,0,0.021512000262737273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,1,128,1,fp8,fp8,0,0.021652799844741822
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,2,128,1,float16,float16,0,0.021198399364948273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,2,128,1,float16,fp8,0,0.021806399524211883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,2,128,1,fp8,fp8,0,0.02173600047826767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,4,128,1,float16,float16,0,0.021227200329303742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,4,128,1,float16,fp8,0,0.021695999801158904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,4,128,1,fp8,fp8,0,0.021775999665260316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,8,128,1,float16,float16,0,0.02126239985227585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,8,128,1,float16,fp8,0,0.022014400362968443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,8,128,1,fp8,fp8,0,0.022121599316596983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,1,128,1,float16,float16,0,0.1642016053199768
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,1,128,1,float16,fp8,0,0.1755776047706604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,1,128,1,fp8,fp8,0,0.177673602104187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,2,128,1,float16,float16,0,0.1713744044303894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,2,128,1,float16,fp8,0,0.185808002948761
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,2,128,1,fp8,fp8,0,0.18754559755325317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,4,128,1,float16,float16,0,0.19118080139160157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,4,128,1,float16,fp8,0,0.2043839931488037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,4,128,1,fp8,fp8,0,0.20356640815734864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,8,128,1,float16,float16,0,0.2580431938171387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,8,128,1,float16,fp8,0,0.2673104047775269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,8,128,1,fp8,fp8,0,0.267303991317749
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,48,128,1,float16,float16,0,0.4491407871246338
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,48,128,1,float16,fp8,0,0.46996321678161623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,1,128,1,float16,float16,0,0.09208800196647644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,48,128,1,fp8,fp8,0,0.4698624134063721
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,1,128,1,float16,fp8,0,0.09984319806098937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,1,128,1,fp8,fp8,0,0.09961119890213013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,2,128,1,float16,float16,0,0.09795359969139099
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,2,128,1,float16,fp8,0,0.10517599582672119
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,2,128,1,fp8,fp8,0,0.10531840324401856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,8,128,1,fp8,fp8,0,0.14713120460510254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,4,128,1,float16,float16,0,0.1060528039932251
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,4,128,1,float16,fp8,0,0.11348799467086793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,4,128,1,fp8,fp8,0,0.112881600856781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,8,128,1,float16,float16,0,0.13978879451751708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,8,128,1,float16,fp8,0,0.1468783974647522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,48,128,1,float16,float16,0,0.23498399257659913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,48,128,1,float16,fp8,0,0.24696478843688965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,1,128,1,float16,float16,0,0.0597536027431488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,48,128,1,fp8,fp8,0,0.24693119525909424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,1,128,1,float16,fp8,0,0.060540801286697386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,1,128,1,fp8,fp8,0,0.05931680202484131
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,2,128,1,float16,float16,0,0.06146079897880554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,2,128,1,float16,fp8,0,0.05958719849586487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,2,128,1,fp8,fp8,0,0.06026560068130493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,4,128,1,float16,float16,0,0.06613119840621948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,4,128,1,float16,fp8,0,0.06765440106391907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,4,128,1,fp8,fp8,0,0.06700639724731446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,8,128,1,float16,float16,0,0.08254719972610473
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,8,128,1,float16,fp8,0,0.08377280235290527
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,8,128,1,fp8,fp8,0,0.08339040279388428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,48,128,1,float16,float16,0,0.12922240495681764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,48,128,1,float16,fp8,0,0.13179359436035157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,1,128,1,float16,float16,0,0.03346560001373291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,48,128,1,fp8,fp8,0,0.13192800283432007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,1,128,1,float16,fp8,0,0.035132798552513125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,1,128,1,fp8,fp8,0,0.035339200496673585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,2,128,1,float16,float16,0,0.034246399998664856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,2,128,1,float16,fp8,0,0.03526560068130493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,8,128,1,float16,fp8,0,0.04527199864387512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,2,128,1,fp8,fp8,0,0.03561280071735382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,4,128,1,float16,float16,0,0.035569599270820616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,4,128,1,float16,fp8,0,0.03630239963531494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,4,128,1,fp8,fp8,0,0.036078399419784545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,8,128,1,float16,float16,0,0.045895999670028685
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,8,128,1,fp8,fp8,0,0.04646719992160797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,48,128,1,float16,float16,0,0.07003039717674256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,48,128,1,float16,fp8,0,0.06766560077667236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,48,128,1,fp8,fp8,0,0.06704319715499878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,1,128,1,float16,float16,0,0.02804799973964691
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,1,128,1,float16,fp8,0,0.02949439883232117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,1,128,1,fp8,fp8,0,0.029308798909187316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,2,128,1,float16,float16,0,0.028067201375961304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,2,128,1,float16,fp8,0,0.029267200827598573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,2,128,1,fp8,fp8,0,0.02953599989414215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,4,128,1,float16,float16,0,0.02810719907283783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,4,128,1,float16,fp8,0,0.02942720055580139
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,4,128,1,fp8,fp8,0,0.029476800560951234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,8,128,1,float16,float16,0,0.02853280007839203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,1,128,1,float16,float16,0,0.02255840003490448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,8,128,1,float16,fp8,0,0.03028160035610199
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,1,128,1,float16,fp8,0,0.023052799701690673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,8,128,1,fp8,fp8,0,0.030124801397323608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,48,128,1,float16,float16,0,0.03895840048789978
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,48,128,1,float16,fp8,0,0.040803200006484984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,48,128,1,fp8,fp8,0,0.04051840007305145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,4,128,1,float16,fp8,0,0.023391999304294586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,1,128,1,fp8,fp8,0,0.023358400166034698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,2,128,1,float16,float16,0,0.022251200675964356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,2,128,1,float16,fp8,0,0.023019200563430785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,2,128,1,fp8,fp8,0,0.023268799483776092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,4,128,1,float16,float16,0,0.022526399791240694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,4,128,1,fp8,fp8,0,0.02326720058917999
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,8,128,1,float16,float16,0,0.02266719937324524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,8,128,1,float16,fp8,0,0.023388800024986268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,8,128,1,fp8,fp8,0,0.02311359941959381
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,48,128,1,float16,float16,0,0.02789439857006073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,48,128,1,float16,fp8,0,0.029264000058174134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,48,128,1,fp8,fp8,0,0.029627200961112977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,1,128,1,float16,float16,0,0.021265600621700288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,1,128,1,float16,fp8,0,0.02239679992198944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,1,128,1,fp8,fp8,0,0.022251200675964356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,2,128,1,float16,float16,0,0.02141599953174591
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,2,128,1,float16,fp8,0,0.02221599966287613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,2,128,1,fp8,fp8,0,0.022283199429512023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,4,128,1,float16,float16,0,0.02160799950361252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,4,128,1,float16,fp8,0,0.022155199944972993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,4,128,1,fp8,fp8,0,0.022324800491333008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,8,128,1,float16,float16,0,0.02151840031147003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,8,128,1,float16,fp8,0,0.022441600263118745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,8,128,1,fp8,fp8,0,0.022327999770641326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,2,128,1,float16,float16,0,0.02115039974451065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,48,128,1,float16,float16,0,0.021700799465179443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,48,128,1,float16,fp8,0,0.022355200350284578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,48,128,1,fp8,fp8,0,0.022735999524593355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,1,128,1,float16,float16,0,0.020715199410915375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,1,128,1,float16,fp8,0,0.021622399985790252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,1,128,1,fp8,fp8,0,0.021873599290847777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,2,128,1,float16,fp8,0,0.02165440022945404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,2,128,1,fp8,fp8,0,0.021831999719142913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,4,128,1,float16,float16,0,0.02093919962644577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,4,128,1,float16,fp8,0,0.021958400309085847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,4,128,1,fp8,fp8,0,0.02223999947309494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,8,128,1,float16,float16,0,0.02090719938278198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,8,128,1,float16,fp8,0,0.021844799816608428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,8,128,1,fp8,fp8,0,0.022252799570560457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,48,128,1,float16,float16,0,0.021054400503635405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,48,128,1,float16,fp8,0,0.022168000042438508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,48,128,1,fp8,fp8,0,0.021598400175571443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,1,128,1,float16,float16,0,0.020552000403404234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,1,128,1,float16,fp8,0,0.02178560048341751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,1,128,1,fp8,fp8,0,0.021219199895858763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,2,128,1,float16,float16,0,0.020790399610996248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,2,128,1,float16,fp8,0,0.021555200219154358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,2,128,1,fp8,fp8,0,0.021427200734615327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,4,128,1,float16,float16,0,0.0208639994263649
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,1,128,1,float16,float16,0,0.09349600076675416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,4,128,1,float16,fp8,0,0.02141280025243759
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,4,128,1,fp8,fp8,0,0.02160640060901642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,8,128,1,float16,float16,0,0.020883199572563172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,8,128,1,float16,fp8,0,0.021673600375652313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,8,128,1,fp8,fp8,0,0.021510399878025055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,4,128,1,float16,float16,0,0.12191840410232543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,4,128,1,float16,fp8,0,0.12979999780654908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,1,128,1,float16,fp8,0,0.10065439939498902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,1,128,1,fp8,fp8,0,0.10061119794845581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,2,128,1,float16,float16,0,0.0988431990146637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,2,128,1,float16,fp8,0,0.10657119750976562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,2,128,1,fp8,fp8,0,0.10682239532470703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,4,128,1,fp8,fp8,0,0.12989920377731323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,8,128,1,float16,float16,0,0.14166079759597777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,8,128,1,float16,fp8,0,0.14889919757843018
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,8,128,1,fp8,fp8,0,0.14957120418548583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,48,128,1,float16,float16,0,0.3274528026580811
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,48,128,1,float16,fp8,0,0.35459840297698975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,1,128,1,float16,float16,0,0.06025760173797608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,48,128,1,fp8,fp8,0,0.35381920337677003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,1,128,1,float16,fp8,0,0.06002879738807678
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,1,128,1,fp8,fp8,0,0.06071680188179016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,2,128,1,float16,float16,0,0.06107680201530456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,2,128,1,float16,fp8,0,0.061900800466537474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,2,128,1,fp8,fp8,0,0.06125119924545288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,4,128,1,float16,float16,0,0.07489920258522034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,4,128,1,float16,fp8,0,0.07575839757919312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,4,128,1,fp8,fp8,0,0.07609599828720093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,8,128,1,float16,float16,0,0.08369119763374329
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,8,128,1,float16,fp8,0,0.08590720295906067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,8,128,1,fp8,fp8,0,0.0853007972240448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,48,128,1,float16,float16,0,0.17479039430618287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,48,128,1,float16,fp8,0,0.18880640268325805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,48,128,1,fp8,fp8,0,0.18896960020065307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,1,128,1,float16,float16,0,0.033878400921821594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,1,128,1,float16,fp8,0,0.03559040129184723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,1,128,1,fp8,fp8,0,0.035579198598861696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,2,128,1,float16,float16,0,0.03419199883937836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,2,128,1,float16,fp8,0,0.03580000102519989
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,2,128,1,fp8,fp8,0,0.03595199882984161
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,4,128,1,float16,float16,0,0.03855839967727661
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,4,128,1,float16,fp8,0,0.040670400857925414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,4,128,1,fp8,fp8,0,0.04065600037574768
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,8,128,1,float16,float16,0,0.046060800552368164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,8,128,1,float16,fp8,0,0.04514879882335663
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,8,128,1,fp8,fp8,0,0.04512479901313782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,48,128,1,float16,float16,0,0.09237279891967773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,48,128,1,float16,fp8,0,0.09469760060310364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,48,128,1,fp8,fp8,0,0.0949072003364563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,1,128,1,float16,float16,0,0.028331199288368226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,1,128,1,float16,fp8,0,0.029572799801826477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,1,128,1,fp8,fp8,0,0.02974080145359039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,2,128,1,float16,float16,0,0.02843039929866791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,2,128,1,float16,fp8,0,0.029603201150894164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,2,128,1,fp8,fp8,0,0.02969760000705719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,4,128,1,float16,float16,0,0.028255999088287354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,4,128,1,float16,fp8,0,0.029609599709510805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,4,128,1,fp8,fp8,0,0.029553601145744325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,8,128,1,float16,float16,0,0.029206401109695433
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,8,128,1,float16,fp8,0,0.03049280047416687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,8,128,1,fp8,fp8,0,0.030452799797058106
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,48,128,1,float16,float16,0,0.05023999810218811
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,48,128,1,float16,fp8,0,0.0543615996837616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,48,128,1,fp8,fp8,0,0.05410879850387573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,1,128,1,float16,float16,0,0.022592000663280487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,1,128,1,float16,fp8,0,0.023108799755573273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,1,128,1,fp8,fp8,0,0.023124800622463228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,8,128,1,float16,float16,0,0.022385600209236144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,2,128,1,float16,float16,0,0.022951999306678773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,2,128,1,float16,fp8,0,0.023550400137901308
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,2,128,1,fp8,fp8,0,0.023668800294399262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,4,128,1,float16,float16,0,0.02220959961414337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,4,128,1,float16,fp8,0,0.022891199588775633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,4,128,1,fp8,fp8,0,0.02319519966840744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,8,128,1,float16,fp8,0,0.023201599717140198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,8,128,1,fp8,fp8,0,0.02311359941959381
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,48,128,1,float16,float16,0,0.03198400139808655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,48,128,1,float16,fp8,0,0.03396160006523132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,48,128,1,fp8,fp8,0,0.03446080088615418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,1,128,1,float16,float16,0,0.021505600214004515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,1,128,1,float16,fp8,0,0.02208320051431656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,1,128,1,fp8,fp8,0,0.022419199347496033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,2,128,1,float16,float16,0,0.021652799844741822
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,2,128,1,float16,fp8,0,0.022308799624443054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,2,128,1,fp8,fp8,0,0.02213920056819916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,4,128,1,float16,float16,0,0.021566399931907655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,4,128,1,float16,fp8,0,0.022598400712013245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,4,128,1,fp8,fp8,0,0.0224031999707222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,8,128,1,float16,float16,0,0.02138399928808212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,8,128,1,float16,fp8,0,0.022268800437450408
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,8,128,1,fp8,fp8,0,0.022342400252819063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,48,128,1,float16,float16,0,0.025329598784446718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,48,128,1,float16,fp8,0,0.026787200570106508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,48,128,1,fp8,fp8,0,0.026876801252365114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,1,128,1,float16,float16,0,0.02089280039072037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,1,128,1,float16,fp8,0,0.02179040014743805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,1,128,1,fp8,fp8,0,0.02165600061416626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,8,128,1,float16,float16,0,0.021116800606250763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,2,128,1,float16,float16,0,0.021315200626850127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,2,128,1,float16,fp8,0,0.021742400527000428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,2,128,1,fp8,fp8,0,0.021784000098705292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,4,128,1,float16,float16,0,0.021184000372886657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,4,128,1,float16,fp8,0,0.022592000663280487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,4,128,1,fp8,fp8,0,0.02221439927816391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,8,128,1,float16,fp8,0,0.022111999988555908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,8,128,1,fp8,fp8,0,0.02210880070924759
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,48,128,1,float16,float16,0,0.021279999613761903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,48,128,1,float16,fp8,0,0.02181120067834854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,48,128,1,fp8,fp8,0,0.021886399388313292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,1,128,1,float16,float16,0,0.02059199959039688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,1,128,1,float16,fp8,0,0.02149759978055954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,1,128,1,fp8,fp8,0,0.021631999313831328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,2,128,1,float16,float16,0,0.020403200387954713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,2,128,1,float16,fp8,0,0.021401600539684297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,2,128,1,fp8,fp8,0,0.021529600024223328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,4,128,1,float16,float16,0,0.021038399636745454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,4,128,1,float16,fp8,0,0.021324799954891206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,4,128,1,fp8,fp8,0,0.021534399688243867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,8,128,1,float16,float16,0,0.02074880003929138
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,8,128,1,float16,fp8,0,0.0216048002243042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,1,128,1,fp8,fp8,0,0.02130880057811737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,8,128,1,fp8,fp8,0,0.02126239985227585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,48,128,1,float16,float16,0,0.021096000075340272
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,48,128,1,float16,fp8,0,0.021687999367713928
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,48,128,1,fp8,fp8,0,0.021563200652599333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,1,128,1,float16,float16,0,0.02054399996995926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,1,128,1,float16,fp8,0,0.021328000724315642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,2,128,1,float16,float16,0,0.020615999400615693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,2,128,1,float16,fp8,0,0.021267199516296388
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,2,128,1,fp8,fp8,0,0.021272000670433045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,4,128,1,float16,float16,0,0.020417599380016326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,4,128,1,float16,fp8,0,0.02109919935464859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,4,128,1,fp8,fp8,0,0.021054400503635405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,8,128,1,float16,float16,0,0.020315200090408325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,8,128,1,float16,fp8,0,0.021209600567817687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,8,128,1,fp8,fp8,0,0.021433599293231964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,1,128,1,float16,fp8,0,30.985598754882812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,1,128,1,fp8,fp8,0,30.748617553710936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,2,128,1,fp8,fp8,0,31.4661865234375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,2,128,1,float16,fp8,0,31.85816650390625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,1,128,1,float16,float16,0,38.37955932617187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,2,128,1,float16,float16,0,38.10728149414062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,4,128,1,float16,fp8,0,31.566607666015624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,4,128,1,float16,float16,0,38.81653747558594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,40,128,1,float16,fp8,0,18.664828491210937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,1,128,1,float16,float16,0,18.55623016357422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,40,128,1,fp8,fp8,0,19.527381896972656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,40,128,1,float16,float16,0,21.587985229492187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,4,128,1,fp8,fp8,0,31.937152099609374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,8,128,1,float16,fp8,0,32.579043579101565
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,8,128,1,fp8,fp8,0,33.14866638183594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,8,128,1,float16,float16,0,38.35848693847656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,1,128,1,float16,fp8,0,16.29266815185547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,1,128,1,fp8,fp8,0,16.010382080078124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,2,128,1,float16,fp8,0,16.54908447265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,2,128,1,float16,float16,0,18.67239990234375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,2,128,1,fp8,fp8,0,17.104559326171874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,4,128,1,float16,fp8,0,17.61707458496094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,4,128,1,float16,float16,0,20.322412109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,4,128,1,fp8,fp8,0,16.52342071533203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,40,128,1,float16,float16,0,11.633258819580078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,40,128,1,float16,fp8,0,9.156476593017578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,8,128,1,float16,fp8,0,16.68060302734375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,8,128,1,fp8,fp8,0,16.431826782226562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,40,128,1,fp8,fp8,0,9.267151641845704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,8,128,1,float16,float16,0,20.0951904296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,1,128,1,float16,fp8,0,7.710692596435547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,1,128,1,float16,float16,0,9.139537811279297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,1,128,1,fp8,fp8,0,7.967467498779297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,2,128,1,float16,fp8,0,7.9556434631347654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,2,128,1,float16,float16,0,9.481095886230468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,2,128,1,fp8,fp8,0,7.89306869506836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,4,128,1,float16,fp8,0,8.331387329101563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,4,128,1,fp8,fp8,0,7.872232055664062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,4,128,1,float16,float16,0,9.701265716552735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,8,128,1,float16,float16,0,9.66239013671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,8,128,1,float16,fp8,0,8.048677062988281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,40,128,1,float16,float16,0,5.4518272399902346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,40,128,1,float16,fp8,0,4.626871871948242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,8,128,1,fp8,fp8,0,8.131617736816406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,40,128,1,fp8,fp8,0,4.665614318847656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,1,128,1,float16,fp8,0,4.036630249023437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,1,128,1,float16,float16,0,5.091057586669922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,1,128,1,fp8,fp8,0,4.206379318237305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,2,128,1,float16,float16,0,4.7407585144042965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,2,128,1,fp8,fp8,0,4.120211029052735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,2,128,1,float16,fp8,0,4.296929550170899
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,4,128,1,float16,fp8,0,4.262432098388672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,4,128,1,fp8,fp8,0,4.232896041870117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,4,128,1,float16,float16,0,4.858318328857422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,8,128,1,float16,float16,0,5.035855865478515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,8,128,1,float16,fp8,0,4.313391876220703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,8,128,1,fp8,fp8,0,4.029555130004883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,1,128,1,float16,fp8,0,18.12634735107422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,1,128,1,fp8,fp8,0,18.798507690429688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,2,128,1,float16,fp8,0,18.537733459472655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,2,128,1,fp8,fp8,0,17.94404296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,1,128,1,float16,float16,0,21.741737365722656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,4,128,1,float16,fp8,0,19.07136688232422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,2,128,1,float16,float16,0,22.23612823486328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,4,128,1,float16,float16,0,21.6273193359375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,40,128,1,float16,fp8,0,11.008857727050781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,40,128,1,float16,float16,0,13.006837463378906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,40,128,1,fp8,fp8,0,11.240001678466797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,1,128,1,float16,float16,0,11.167375946044922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,4,128,1,fp8,fp8,0,18.891026306152344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,8,128,1,float16,fp8,0,18.976058959960938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,8,128,1,fp8,fp8,0,19.093699645996093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,1,128,1,float16,fp8,0,9.13597412109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,8,128,1,float16,float16,0,22.932147216796874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,1,128,1,fp8,fp8,0,9.279390716552735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,2,128,1,float16,fp8,0,9.406935882568359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,2,128,1,fp8,fp8,0,9.318167877197265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,2,128,1,float16,float16,0,11.78082275390625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,4,128,1,float16,fp8,0,9.230043029785156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,4,128,1,float16,float16,0,11.583017730712891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,4,128,1,fp8,fp8,0,9.352881622314452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,40,128,1,float16,float16,0,6.611246490478516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,40,128,1,float16,fp8,0,5.6140495300292965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,8,128,1,float16,fp8,0,9.480478668212891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,8,128,1,float16,float16,0,11.493433380126953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,8,128,1,fp8,fp8,0,9.35294418334961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,40,128,1,fp8,fp8,0,5.45930061340332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,1,128,1,float16,float16,0,5.417115020751953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,1,128,1,float16,fp8,0,4.627401733398438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,1,128,1,fp8,fp8,0,4.454921722412109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,2,128,1,float16,fp8,0,4.577950286865234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,2,128,1,float16,float16,0,5.612766265869141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,2,128,1,fp8,fp8,0,4.7076255798339846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,4,128,1,float16,fp8,0,4.547262573242188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,4,128,1,float16,float16,0,5.229046249389649
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,4,128,1,fp8,fp8,0,4.539907073974609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,40,128,1,float16,float16,0,3.230379104614258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,8,128,1,float16,fp8,0,4.649923324584961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,40,128,1,float16,fp8,0,2.8088687896728515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,8,128,1,float16,float16,0,6.0256702423095705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,8,128,1,fp8,fp8,0,4.684550476074219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,40,128,1,fp8,fp8,0,3.026487922668457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,1,128,1,float16,float16,0,2.602838325500488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,1,128,1,float16,fp8,0,2.3357919692993163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,1,128,1,fp8,fp8,0,2.2254928588867187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,2,128,1,float16,float16,0,2.983417510986328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,2,128,1,float16,fp8,0,2.454196739196777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,2,128,1,fp8,fp8,0,2.2536767959594726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,4,128,1,float16,float16,0,2.6309024810791017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,4,128,1,float16,fp8,0,2.3948127746582033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,4,128,1,fp8,fp8,0,2.338417625427246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,8,128,1,float16,float16,0,2.848251152038574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,8,128,1,fp8,fp8,0,2.3144351959228517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,8,128,1,float16,fp8,0,2.599017524719238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,1,128,1,fp8,fp8,0,12.607577514648437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,1,128,1,float16,fp8,0,13.131759643554688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,2,128,1,float16,fp8,0,12.738916778564453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,2,128,1,fp8,fp8,0,12.522126770019531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,1,128,1,float16,float16,0,15.690010070800781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,4,128,1,float16,fp8,0,13.717710876464844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,2,128,1,float16,float16,0,15.858061218261719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,4,128,1,float16,float16,0,15.067393493652343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,40,128,1,float16,float16,0,9.45804672241211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,1,128,1,float16,float16,0,7.591960144042969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,40,128,1,float16,fp8,0,8.7781982421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,40,128,1,fp8,fp8,0,8.135291290283202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,4,128,1,fp8,fp8,0,13.444955444335937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,8,128,1,fp8,fp8,0,13.043684387207032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,8,128,1,float16,fp8,0,13.63441162109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,8,128,1,float16,float16,0,15.50859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,1,128,1,float16,fp8,0,6.21336784362793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,1,128,1,fp8,fp8,0,6.375779342651367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,2,128,1,float16,fp8,0,6.582592010498047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,2,128,1,float16,float16,0,8.109657287597656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,2,128,1,fp8,fp8,0,6.447071838378906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,4,128,1,float16,fp8,0,6.487651062011719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,4,128,1,float16,float16,0,8.263887786865235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,4,128,1,fp8,fp8,0,6.5195152282714846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,40,128,1,float16,fp8,0,4.041286468505859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,40,128,1,float16,float16,0,4.6983184814453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,8,128,1,float16,fp8,0,7.038703918457031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,8,128,1,fp8,fp8,0,6.760572814941407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,40,128,1,fp8,fp8,0,4.12032470703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,8,128,1,float16,float16,0,7.981743621826172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,1,128,1,float16,fp8,0,3.1774816513061523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,1,128,1,float16,float16,0,4.230340957641602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,1,128,1,fp8,fp8,0,3.129604721069336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,2,128,1,float16,fp8,0,3.2092735290527346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,2,128,1,float16,float16,0,3.909543991088867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,2,128,1,fp8,fp8,0,3.216708755493164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,4,128,1,float16,fp8,0,3.2506385803222657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,4,128,1,float16,float16,0,3.8105743408203123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,4,128,1,fp8,fp8,0,3.3547454833984376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,40,128,1,float16,fp8,0,2.032369613647461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,8,128,1,float16,float16,0,4.03974723815918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,40,128,1,float16,float16,0,2.343668746948242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,8,128,1,float16,fp8,0,3.2823680877685546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,1,128,1,float16,fp8,0,1.609939193725586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,40,128,1,fp8,fp8,0,2.024192047119141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,1,128,1,float16,float16,0,1.8561296463012695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,8,128,1,fp8,fp8,0,3.2823406219482423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,1,128,1,fp8,fp8,0,1.607467269897461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,2,128,1,float16,float16,0,1.9023136138916015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,2,128,1,float16,fp8,0,1.676767921447754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,2,128,1,fp8,fp8,0,1.5855135917663574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,4,128,1,float16,fp8,0,1.6060400009155273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,4,128,1,float16,float16,0,1.7844623565673827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,4,128,1,fp8,fp8,0,1.7401311874389649
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,8,128,1,float16,float16,0,1.7901424407958983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,8,128,1,float16,fp8,0,1.7351823806762696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,8,128,1,fp8,fp8,0,1.6833824157714843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,1,128,1,float16,fp8,0,16.814585876464843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,1,128,1,fp8,fp8,0,16.693670654296874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,2,128,1,float16,fp8,0,16.589649963378907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,2,128,1,fp8,fp8,0,16.64391326904297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,4,128,1,float16,fp8,0,16.642250061035156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,1,128,1,float16,float16,0,19.983656311035155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,2,128,1,float16,float16,0,19.877606201171876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,4,128,1,float16,float16,0,20.755238342285157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,40,128,1,float16,float16,0,12.969857788085937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,40,128,1,float16,fp8,0,11.132134246826173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,40,128,1,fp8,fp8,0,11.020798492431641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,1,128,1,float16,float16,0,9.99324951171875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,4,128,1,fp8,fp8,0,17.453877258300782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,8,128,1,float16,fp8,0,17.762049865722656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,8,128,1,fp8,fp8,0,18.10210266113281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,8,128,1,float16,float16,0,21.536318969726562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,1,128,1,float16,fp8,0,8.429627227783204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,1,128,1,fp8,fp8,0,8.818447875976563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,2,128,1,float16,fp8,0,8.663992309570313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,2,128,1,float16,float16,0,10.403467559814453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,2,128,1,fp8,fp8,0,8.29773406982422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,4,128,1,float16,fp8,0,8.450411224365235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,4,128,1,float16,float16,0,10.386182403564453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,4,128,1,fp8,fp8,0,8.358081817626953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,40,128,1,float16,float16,0,6.241604614257812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,40,128,1,float16,fp8,0,5.458833694458008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,8,128,1,float16,fp8,0,9.138916778564454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,40,128,1,fp8,fp8,0,5.433966445922851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,8,128,1,fp8,fp8,0,9.088006591796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,8,128,1,float16,float16,0,10.401622772216797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,1,128,1,float16,float16,0,4.974031829833985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,1,128,1,float16,fp8,0,4.11688461303711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,1,128,1,fp8,fp8,0,4.262980651855469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,2,128,1,float16,fp8,0,4.1970977783203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,2,128,1,fp8,fp8,0,4.198265457153321
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,2,128,1,float16,float16,0,5.527423858642578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,4,128,1,float16,float16,0,4.717489624023438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,4,128,1,float16,fp8,0,4.300425720214844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,4,128,1,fp8,fp8,0,4.18852653503418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,40,128,1,float16,float16,0,3.1373104095458983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,8,128,1,float16,fp8,0,4.3559825897216795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,8,128,1,float16,float16,0,5.406492614746094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,40,128,1,fp8,fp8,0,2.913105583190918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,40,128,1,float16,fp8,0,3.080076789855957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,8,128,1,fp8,fp8,0,4.4512382507324215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,1,128,1,float16,float16,0,2.2802831649780275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,1,128,1,float16,fp8,0,2.0446128845214844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,1,128,1,fp8,fp8,0,2.3079967498779297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,2,128,1,float16,float16,0,2.3044240951538084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,2,128,1,fp8,fp8,0,2.0681856155395506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,2,128,1,float16,fp8,0,2.258996772766113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,4,128,1,float16,fp8,0,2.1163055419921877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,4,128,1,float16,float16,0,2.538808059692383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,4,128,1,fp8,fp8,0,2.2652111053466797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,8,128,1,float16,float16,0,2.5482240676879884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,40,128,1,float16,float16,0,1.4825504302978516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,40,128,1,float16,fp8,0,1.3891103744506836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,8,128,1,float16,fp8,0,2.189569664001465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,40,128,1,fp8,fp8,0,1.3910016059875487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,1,128,1,float16,fp8,0,1.0630208015441895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,1,128,1,float16,float16,0,1.328206443786621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,8,128,1,fp8,fp8,0,2.3247264862060546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,1,128,1,fp8,fp8,0,1.1890000343322753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,2,128,1,float16,float16,0,1.1323552131652832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,2,128,1,float16,fp8,0,1.0495344161987306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,2,128,1,fp8,fp8,0,1.049078369140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,4,128,1,float16,float16,0,1.1467455863952636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,4,128,1,float16,fp8,0,1.068006420135498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,4,128,1,fp8,fp8,0,1.1861727714538575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,8,128,1,float16,float16,0,1.1927712440490723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,8,128,1,float16,fp8,0,1.2311391830444336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,8,128,1,fp8,fp8,0,1.1936256408691406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,1,128,1,float16,fp8,0,9.749311828613282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,1,128,1,fp8,fp8,0,9.778729248046876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,2,128,1,fp8,fp8,0,9.571046447753906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,2,128,1,float16,fp8,0,10.154682922363282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,1,128,1,float16,float16,0,11.342559814453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,4,128,1,float16,fp8,0,9.855379486083985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,2,128,1,float16,float16,0,11.330435180664063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,4,128,1,float16,float16,0,11.743572998046876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,40,128,1,float16,fp8,0,6.707132720947266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,1,128,1,float16,float16,0,5.668379211425782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,40,128,1,fp8,fp8,0,6.798182678222656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,40,128,1,float16,float16,0,7.802339172363281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,4,128,1,fp8,fp8,0,10.227200317382813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,8,128,1,float16,fp8,0,10.167467498779297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,8,128,1,fp8,fp8,0,11.243638610839843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,8,128,1,float16,float16,0,11.957054138183594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,1,128,1,float16,fp8,0,4.685363388061523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,1,128,1,fp8,fp8,0,4.843545532226562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,2,128,1,float16,fp8,0,4.870572662353515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,2,128,1,float16,float16,0,5.885580825805664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,2,128,1,fp8,fp8,0,4.914004898071289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,4,128,1,float16,float16,0,6.073260879516601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,4,128,1,float16,fp8,0,5.237742233276367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,4,128,1,fp8,fp8,0,4.937255859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,40,128,1,float16,float16,0,3.7734142303466798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,8,128,1,float16,fp8,0,5.106014251708984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,40,128,1,float16,fp8,0,3.422742462158203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,8,128,1,float16,float16,0,6.182003021240234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,8,128,1,fp8,fp8,0,5.235044860839844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,1,128,1,float16,fp8,0,2.357372856140137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,1,128,1,float16,float16,0,2.931380844116211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,40,128,1,fp8,fp8,0,3.7220481872558593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,1,128,1,fp8,fp8,0,2.467353630065918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,2,128,1,float16,fp8,0,2.4589056015014648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,2,128,1,float16,float16,0,2.8185583114624024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,2,128,1,fp8,fp8,0,2.4713855743408204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,4,128,1,float16,float16,0,2.9186752319335936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,4,128,1,float16,fp8,0,2.519318389892578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,4,128,1,fp8,fp8,0,2.790990447998047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,40,128,1,float16,fp8,0,1.7215408325195312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,40,128,1,float16,float16,0,2.1100576400756834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,8,128,1,float16,fp8,0,2.622763252258301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,8,128,1,float16,float16,0,3.0004512786865236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,1,128,1,float16,float16,0,1.3091792106628417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,8,128,1,fp8,fp8,0,2.596638488769531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,40,128,1,fp8,fp8,0,1.9198240280151366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,1,128,1,float16,fp8,0,1.2928079605102538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,1,128,1,fp8,fp8,0,1.1897184371948242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,2,128,1,float16,float16,0,1.312820816040039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,2,128,1,float16,fp8,0,1.2194047927856446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,2,128,1,fp8,fp8,0,1.3979455947875976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,4,128,1,float16,float16,0,1.389070415496826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,4,128,1,float16,fp8,0,1.2438032150268554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,4,128,1,fp8,fp8,0,1.2453472137451171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,8,128,1,float16,float16,0,1.41636323928833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,8,128,1,float16,fp8,0,1.3276656150817872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,40,128,1,float16,float16,0,0.953763198852539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,1,128,1,float16,float16,0,0.6452943801879882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,8,128,1,fp8,fp8,0,1.3771183967590332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,40,128,1,float16,fp8,0,0.9577072143554688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,40,128,1,fp8,fp8,0,0.8756239891052247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,1,128,1,float16,fp8,0,0.670966386795044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,2,128,1,float16,float16,0,0.6595488071441651
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,1,128,1,fp8,fp8,0,0.6255184173583984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,2,128,1,fp8,fp8,0,0.641758394241333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,2,128,1,float16,fp8,0,0.6215231895446778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,4,128,1,float16,float16,0,0.7043871879577637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,4,128,1,float16,fp8,0,0.6458672046661377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,4,128,1,fp8,fp8,0,0.6662847995758057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,8,128,1,float16,float16,0,0.7012080192565918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,8,128,1,float16,fp8,0,0.6701312065124512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,8,128,1,fp8,fp8,0,0.6753007888793945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,1,128,1,float16,fp8,0,8.868656158447266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,1,128,1,fp8,fp8,0,8.943638610839844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,2,128,1,float16,fp8,0,9.041883087158203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,2,128,1,fp8,fp8,0,9.012073516845703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,1,128,1,float16,float16,0,10.444491577148437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,2,128,1,float16,float16,0,10.933217620849609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,4,128,1,float16,float16,0,10.82577896118164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,4,128,1,float16,fp8,0,9.32647705078125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,1,128,1,float16,float16,0,4.871543884277344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,40,128,1,float16,float16,0,7.998945617675782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,4,128,1,fp8,fp8,0,9.339002990722657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,40,128,1,float16,fp8,0,7.434588623046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,40,128,1,fp8,fp8,0,7.208841705322266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,8,128,1,float16,fp8,0,9.867079925537109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,8,128,1,fp8,fp8,0,9.8606689453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,8,128,1,float16,float16,0,11.771675109863281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,1,128,1,float16,fp8,0,4.613703918457031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,1,128,1,fp8,fp8,0,4.437233734130859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,2,128,1,fp8,fp8,0,4.517814254760742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,2,128,1,float16,fp8,0,4.753339385986328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,2,128,1,float16,float16,0,5.711524963378906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,4,128,1,float16,fp8,0,4.654504013061524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,4,128,1,float16,float16,0,5.553007888793945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,4,128,1,fp8,fp8,0,4.733299255371094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,40,128,1,float16,float16,0,4.037115097045898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,40,128,1,float16,fp8,0,3.6837841033935548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,8,128,1,float16,fp8,0,4.95281753540039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,8,128,1,fp8,fp8,0,4.966796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,8,128,1,float16,float16,0,5.616764831542969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,1,128,1,float16,float16,0,2.4306575775146486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,40,128,1,fp8,fp8,0,3.5932849884033202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,1,128,1,float16,fp8,0,2.230411148071289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,2,128,1,float16,float16,0,2.467001533508301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,1,128,1,fp8,fp8,0,2.2776079177856445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,2,128,1,fp8,fp8,0,2.274951934814453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,2,128,1,float16,fp8,0,2.261020851135254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,4,128,1,fp8,fp8,0,2.330881690979004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,4,128,1,float16,float16,0,2.561720085144043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,4,128,1,float16,fp8,0,2.483273506164551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,8,128,1,float16,float16,0,2.8888864517211914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,1,128,1,float16,float16,0,1.2540559768676758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,1,128,1,float16,fp8,0,1.1526415824890137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,40,128,1,float16,float16,0,1.9482336044311523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,40,128,1,float16,fp8,0,1.8245616912841798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,40,128,1,fp8,fp8,0,1.8093183517456055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,1,128,1,fp8,fp8,0,1.3205504417419434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,8,128,1,fp8,fp8,0,2.540999984741211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,8,128,1,float16,fp8,0,2.623259162902832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,2,128,1,float16,fp8,0,1.1520544052124024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,2,128,1,float16,float16,0,1.2958671569824218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,2,128,1,fp8,fp8,0,1.2061344146728517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,4,128,1,float16,float16,0,1.2526512145996094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,4,128,1,float16,fp8,0,1.1767007827758789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,4,128,1,fp8,fp8,0,1.2437760353088378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,8,128,1,float16,float16,0,1.3404735565185546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,8,128,1,float16,fp8,0,1.2574272155761719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,1,128,1,float16,float16,0,0.6024991989135742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,40,128,1,float16,float16,0,1.0381855964660645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,8,128,1,fp8,fp8,0,1.2583760261535644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,1,128,1,float16,fp8,0,0.5946383953094483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,40,128,1,float16,fp8,0,0.9703791618347168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,2,128,1,float16,float16,0,0.6177487850189209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,40,128,1,fp8,fp8,0,0.9764495849609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,1,128,1,fp8,fp8,0,0.6131423950195313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,2,128,1,float16,fp8,0,0.6003920078277588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,2,128,1,fp8,fp8,0,0.6491631984710693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,4,128,1,fp8,fp8,0,0.6289023876190185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,4,128,1,float16,float16,0,0.6461440086364746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,4,128,1,float16,fp8,0,0.6237423896789551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,8,128,1,float16,float16,0,0.6726607799530029
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,8,128,1,float16,fp8,0,0.6603328227996826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,8,128,1,fp8,fp8,0,0.6681344032287597
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,40,128,1,float16,float16,0,0.5081855773925781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,40,128,1,float16,fp8,0,0.502627182006836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,1,128,1,float16,float16,0,0.32020320892333987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,40,128,1,fp8,fp8,0,0.4910719871520996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,1,128,1,float16,fp8,0,0.307369589805603
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,1,128,1,fp8,fp8,0,0.30214719772338866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,2,128,1,float16,float16,0,0.32948319911956786
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,2,128,1,float16,fp8,0,0.3144047975540161
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,4,128,1,float16,fp8,0,0.3216912031173706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,4,128,1,float16,float16,0,0.33679680824279784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,4,128,1,fp8,fp8,0,0.32350399494171145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,2,128,1,fp8,fp8,0,0.3161456108093262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,8,128,1,float16,fp8,0,0.3411072015762329
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,8,128,1,float16,float16,0,0.3560431957244873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,8,128,1,fp8,fp8,0,0.3408447980880737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,1,128,1,float16,fp8,0,5.342222213745117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,1,128,1,fp8,fp8,0,5.352758407592773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,1,128,1,float16,float16,0,6.050137710571289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,2,128,1,float16,fp8,0,5.434345626831055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,2,128,1,float16,float16,0,6.095467376708984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,2,128,1,fp8,fp8,0,5.468937683105469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,4,128,1,float16,float16,0,6.379880142211914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,4,128,1,float16,fp8,0,5.6279457092285154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,4,128,1,fp8,fp8,0,5.620841598510742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,40,128,1,float16,float16,0,5.1681873321533205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,8,128,1,float16,fp8,0,6.036054229736328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,8,128,1,fp8,fp8,0,6.032403182983399
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,8,128,1,float16,float16,0,6.847756958007812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,1,128,1,float16,fp8,0,2.6408096313476563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,1,128,1,float16,float16,0,2.939131164550781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,40,128,1,float16,fp8,0,4.775697708129883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,1,128,1,fp8,fp8,0,2.701251220703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,40,128,1,fp8,fp8,0,4.796540832519531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,2,128,1,float16,float16,0,2.965185546875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,2,128,1,float16,fp8,0,2.6922943115234377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,2,128,1,fp8,fp8,0,2.7485328674316407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,4,128,1,float16,float16,0,3.1853408813476562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,4,128,1,float16,fp8,0,2.9658960342407226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,4,128,1,fp8,fp8,0,2.8264591217041017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,40,128,1,float16,fp8,0,2.3844207763671874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,40,128,1,float16,float16,0,2.543092727661133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,8,128,1,float16,fp8,0,3.048348808288574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,1,128,1,float16,float16,0,1.4194128036499023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,8,128,1,fp8,fp8,0,3.1855920791625976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,8,128,1,float16,float16,0,3.360160064697266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,1,128,1,float16,fp8,0,1.3344143867492675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,1,128,1,fp8,fp8,0,1.334996795654297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,40,128,1,fp8,fp8,0,2.4504863739013674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,2,128,1,float16,fp8,0,1.369870376586914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,2,128,1,float16,float16,0,1.450881576538086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,2,128,1,fp8,fp8,0,1.4444992065429687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,4,128,1,float16,float16,0,1.5204272270202637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,4,128,1,fp8,fp8,0,1.4254608154296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,4,128,1,float16,fp8,0,1.516108798980713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,8,128,1,float16,float16,0,1.6247631072998048
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,8,128,1,float16,fp8,0,1.5359487533569336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,8,128,1,fp8,fp8,0,1.5294320106506347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,40,128,1,float16,float16,0,1.316972827911377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,1,128,1,float16,float16,0,0.7215392112731933
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,1,128,1,float16,fp8,0,0.7055136203765869
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,40,128,1,float16,fp8,0,1.2396431922912599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,40,128,1,fp8,fp8,0,1.2353152275085448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,1,128,1,fp8,fp8,0,0.7028416156768799
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,2,128,1,float16,float16,0,0.741267204284668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,2,128,1,float16,fp8,0,0.6957536220550538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,2,128,1,fp8,fp8,0,0.7196239948272705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,4,128,1,float16,float16,0,0.7848944187164306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,4,128,1,float16,fp8,0,0.7302127838134765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,4,128,1,fp8,fp8,0,0.7388591766357422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,1,128,1,float16,float16,0,0.3752496004104614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,8,128,1,fp8,fp8,0,0.8017840385437012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,1,128,1,float16,fp8,0,0.3602368116378784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,40,128,1,fp8,fp8,0,0.6326687812805176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,1,128,1,fp8,fp8,0,0.35523200035095215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,40,128,1,float16,fp8,0,0.6296544075012207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,8,128,1,float16,fp8,0,0.8030240058898925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,40,128,1,float16,float16,0,0.6476223945617676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,8,128,1,float16,float16,0,0.8189248085021973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,2,128,1,float16,float16,0,0.38285601139068604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,2,128,1,float16,fp8,0,0.3718863964080811
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,2,128,1,fp8,fp8,0,0.37023999691009524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,4,128,1,float16,float16,0,0.40064802169799807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,4,128,1,float16,fp8,0,0.3851968050003052
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,4,128,1,fp8,fp8,0,0.387225604057312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,8,128,1,float16,float16,0,0.41188797950744627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,8,128,1,float16,fp8,0,0.4117424011230469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,8,128,1,fp8,fp8,0,0.4111631870269775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,40,128,1,float16,float16,0,0.3509488105773926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,40,128,1,float16,fp8,0,0.32543039321899414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,40,128,1,fp8,fp8,0,0.32687039375305177
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,1,128,1,float16,float16,0,0.20004799365997314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,1,128,1,float16,fp8,0,0.19635039567947388
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,1,128,1,fp8,fp8,0,0.1916416049003601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,2,128,1,float16,float16,0,0.20301918983459472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,2,128,1,float16,fp8,0,0.19599679708480836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,2,128,1,fp8,fp8,0,0.19910080432891847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,4,128,1,float16,float16,0,0.2129040002822876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,4,128,1,float16,fp8,0,0.20367679595947266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,4,128,1,fp8,fp8,0,0.20252480506896972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,8,128,1,float16,float16,0,0.22387359142303467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,8,128,1,float16,fp8,0,0.21305599212646484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,8,128,1,fp8,fp8,0,0.21815838813781738
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,1,128,1,fp8,fp8,0,5.244259262084961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,1,128,1,float16,fp8,0,5.229220962524414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,1,128,1,float16,float16,0,5.745606231689453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,2,128,1,float16,float16,0,5.971633529663086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,2,128,1,fp8,fp8,0,5.396769714355469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,2,128,1,float16,fp8,0,5.372091293334961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,4,128,1,float16,fp8,0,5.693507385253906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,4,128,1,float16,float16,0,6.208947372436524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,4,128,1,fp8,fp8,0,5.671115112304688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,40,128,1,float16,fp8,0,5.374523162841797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,8,128,1,float16,fp8,0,6.253638458251953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,8,128,1,fp8,fp8,0,6.30419692993164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,40,128,1,fp8,fp8,0,5.435704040527344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,8,128,1,float16,float16,0,6.890415954589844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,40,128,1,float16,float16,0,5.701881790161133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,1,128,1,float16,fp8,0,2.6039663314819337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,1,128,1,float16,float16,0,2.839619255065918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,1,128,1,fp8,fp8,0,2.6053535461425783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,2,128,1,float16,fp8,0,2.6982112884521485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,2,128,1,float16,float16,0,2.987723159790039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,2,128,1,fp8,fp8,0,2.7519216537475586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,4,128,1,float16,float16,0,3.0822032928466796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,4,128,1,float16,fp8,0,2.8587200164794924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,4,128,1,fp8,fp8,0,2.8262271881103516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,8,128,1,float16,fp8,0,3.1194831848144533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,8,128,1,float16,float16,0,3.4304927825927733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,40,128,1,float16,fp8,0,2.7192560195922852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,40,128,1,float16,float16,0,2.9495983123779297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,8,128,1,fp8,fp8,0,3.1403104782104494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,1,128,1,float16,float16,0,1.4000960350036622
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,1,128,1,float16,fp8,0,1.309819221496582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,40,128,1,fp8,fp8,0,2.7210464477539062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,1,128,1,fp8,fp8,0,1.3225232124328614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,2,128,1,float16,float16,0,1.436315155029297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,2,128,1,float16,fp8,0,1.3554112434387207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,2,128,1,fp8,fp8,0,1.383456039428711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,4,128,1,float16,float16,0,1.4863231658935547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,4,128,1,float16,fp8,0,1.487006378173828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,4,128,1,fp8,fp8,0,1.4219615936279297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,8,128,1,float16,float16,0,1.6383327484130858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,8,128,1,float16,fp8,0,1.5678576469421386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,1,128,1,float16,float16,0,0.7066703796386719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,1,128,1,float16,fp8,0,0.6684336185455322
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,40,128,1,float16,float16,0,1.4353615760803222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,8,128,1,fp8,fp8,0,1.6756080627441405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,40,128,1,float16,fp8,0,1.402238368988037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,1,128,1,fp8,fp8,0,0.7473983764648438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,40,128,1,fp8,fp8,0,1.4300959587097168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,2,128,1,float16,float16,0,0.7203855991363526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,2,128,1,float16,fp8,0,0.7028543949127197
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,2,128,1,fp8,fp8,0,0.6999631881713867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,4,128,1,float16,float16,0,0.7654751777648926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,4,128,1,float16,fp8,0,0.7437583923339843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,8,128,1,float16,float16,0,0.8582464218139648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,4,128,1,fp8,fp8,0,0.7452079772949218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,8,128,1,float16,fp8,0,0.8268527984619141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,8,128,1,fp8,fp8,0,0.8189696311950684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,40,128,1,float16,float16,0,0.7418144226074219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,40,128,1,float16,fp8,0,0.7148591995239257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,1,128,1,float16,float16,0,0.36700000762939455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,40,128,1,fp8,fp8,0,0.7142303943634033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,1,128,1,float16,fp8,0,0.3578768014907837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,1,128,1,fp8,fp8,0,0.3581968069076538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,2,128,1,float16,float16,0,0.3770335912704468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,2,128,1,float16,fp8,0,0.36683199405670164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,2,128,1,fp8,fp8,0,0.36587839126586913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,4,128,1,float16,float16,0,0.395686411857605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,4,128,1,float16,fp8,0,0.38774240016937256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,4,128,1,fp8,fp8,0,0.3854640007019043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,8,128,1,float16,float16,0,0.43415679931640627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,8,128,1,float16,fp8,0,0.423689603805542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,8,128,1,fp8,fp8,0,0.42187838554382323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,40,128,1,float16,float16,0,0.38674559593200686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,40,128,1,float16,fp8,0,0.3689903974533081
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,40,128,1,fp8,fp8,0,0.3715104103088379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,1,128,1,float16,float16,0,0.19665759801864624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,1,128,1,float16,fp8,0,0.18984479904174806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,1,128,1,fp8,fp8,0,0.19226720333099365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,2,128,1,float16,float16,0,0.20463359355926514
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,2,128,1,float16,fp8,0,0.19664160013198853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,2,128,1,fp8,fp8,0,0.19873119592666627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,4,128,1,float16,float16,0,0.21103041172027587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,4,128,1,float16,fp8,0,0.20611200332641602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,4,128,1,fp8,fp8,0,0.20667200088500975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,8,128,1,float16,float16,0,0.23136320114135742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,8,128,1,float16,fp8,0,0.22483360767364502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,8,128,1,fp8,fp8,0,0.224783992767334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,40,128,1,float16,float16,0,0.21193599700927734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,40,128,1,float16,fp8,0,0.20099999904632568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,40,128,1,fp8,fp8,0,0.20158560276031495
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,1,128,1,float16,float16,0,0.11190719604492187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,4,128,1,float16,float16,0,0.11658400297164917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,1,128,1,float16,fp8,0,0.10261119604110717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,1,128,1,fp8,fp8,0,0.1037071943283081
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,2,128,1,float16,float16,0,0.11147520542144776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,2,128,1,float16,fp8,0,0.10398720502853394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,2,128,1,fp8,fp8,0,0.10585600137710571
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,4,128,1,float16,fp8,0,0.11296800374984742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,4,128,1,fp8,fp8,0,0.1150704026222229
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,8,128,1,float16,float16,0,0.12502559423446655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,8,128,1,float16,fp8,0,0.12218079566955567
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,8,128,1,fp8,fp8,0,0.12357439994812011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,1,128,1,float16,float16,0,3.4017295837402344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,1,128,1,float16,fp8,0,3.255588912963867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,1,128,1,fp8,fp8,0,3.242745590209961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,2,128,1,float16,fp8,0,3.360940933227539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,2,128,1,float16,float16,0,3.6729457855224608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,2,128,1,fp8,fp8,0,3.3546897888183596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,4,128,1,float16,fp8,0,3.591775894165039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,4,128,1,float16,float16,0,3.754719924926758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,4,128,1,fp8,fp8,0,3.573166275024414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,8,128,1,float16,float16,0,4.3745582580566404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,8,128,1,float16,fp8,0,3.9829601287841796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,40,128,1,float16,float16,0,3.918859100341797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,8,128,1,fp8,fp8,0,4.019873428344726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,40,128,1,float16,fp8,0,3.7399520874023438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,40,128,1,fp8,fp8,0,3.8154415130615233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,1,128,1,float16,float16,0,1.6980064392089844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,1,128,1,float16,fp8,0,1.6982816696166991
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,1,128,1,fp8,fp8,0,1.615020751953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,2,128,1,float16,float16,0,1.7704639434814453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,2,128,1,float16,fp8,0,1.6873600006103515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,2,128,1,fp8,fp8,0,1.68590087890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,4,128,1,float16,float16,0,1.870302391052246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,4,128,1,float16,fp8,0,1.8643760681152344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,4,128,1,fp8,fp8,0,1.8035951614379884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,8,128,1,float16,float16,0,2.144318389892578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,1,128,1,float16,float16,0,0.8851136207580567
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,8,128,1,float16,fp8,0,2.001268768310547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,40,128,1,float16,float16,0,1.952907180786133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,8,128,1,fp8,fp8,0,2.1106943130493163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,40,128,1,float16,fp8,0,1.8779119491577148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,1,128,1,float16,fp8,0,0.8280816078186035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,40,128,1,fp8,fp8,0,1.9618415832519531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,1,128,1,fp8,fp8,0,0.8649087905883789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,2,128,1,float16,float16,0,0.9021776199340821
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,2,128,1,float16,fp8,0,0.8480143547058105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,2,128,1,fp8,fp8,0,0.8556943893432617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,4,128,1,float16,float16,0,0.9553296089172363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,4,128,1,float16,fp8,0,0.9080896377563477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,4,128,1,fp8,fp8,0,0.9121040344238281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,8,128,1,float16,float16,0,1.061903953552246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,8,128,1,float16,fp8,0,1.042081642150879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,8,128,1,fp8,fp8,0,1.0422207832336425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,1,128,1,float16,float16,0,0.43523521423339845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,40,128,1,float16,float16,0,0.9818927764892578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,1,128,1,float16,fp8,0,0.4269279956817627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,40,128,1,float16,fp8,0,0.9813296318054199
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,40,128,1,fp8,fp8,0,0.9787504196166992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,1,128,1,fp8,fp8,0,0.4273712158203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,2,128,1,float16,float16,0,0.46335678100585936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,4,128,1,float16,fp8,0,0.47477121353149415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,2,128,1,float16,fp8,0,0.4493391990661621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,2,128,1,fp8,fp8,0,0.44445600509643557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,4,128,1,float16,float16,0,0.4767183780670166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,4,128,1,fp8,fp8,0,0.4787712097167969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,8,128,1,float16,float16,0,0.5473072052001953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,8,128,1,float16,fp8,0,0.52467041015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,8,128,1,fp8,fp8,0,0.532966423034668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,40,128,1,float16,float16,0,0.515556812286377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,40,128,1,float16,fp8,0,0.5045904159545899
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,1,128,1,float16,float16,0,0.22748959064483643
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,1,128,1,float16,fp8,0,0.22849600315093993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,1,128,1,fp8,fp8,0,0.22935841083526612
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,40,128,1,fp8,fp8,0,0.5020607948303223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,2,128,1,float16,float16,0,0.2390064001083374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,2,128,1,float16,fp8,0,0.23736159801483153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,2,128,1,fp8,fp8,0,0.23607521057128905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,4,128,1,float16,float16,0,0.2530064105987549
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,4,128,1,float16,fp8,0,0.24525279998779298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,4,128,1,fp8,fp8,0,0.24735679626464843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,8,128,1,float16,float16,0,0.28684799671173095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,8,128,1,float16,fp8,0,0.27855520248413085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,8,128,1,fp8,fp8,0,0.2797679901123047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,40,128,1,float16,float16,0,0.27464001178741454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,40,128,1,float16,fp8,0,0.2658112049102783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,40,128,1,fp8,fp8,0,0.2623424053192139
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,1,128,1,float16,float16,0,0.1293951988220215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,1,128,1,float16,fp8,0,0.12628799676895142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,1,128,1,fp8,fp8,0,0.12817599773406982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,4,128,1,fp8,fp8,0,0.1366368055343628
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,8,128,1,float16,float16,0,0.1537376046180725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,8,128,1,float16,fp8,0,0.15236639976501465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,2,128,1,float16,float16,0,0.13206880092620848
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,2,128,1,float16,fp8,0,0.1301792025566101
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,2,128,1,fp8,fp8,0,0.12935680150985718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,4,128,1,float16,float16,0,0.1396239995956421
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,4,128,1,float16,fp8,0,0.13830239772796632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,8,128,1,fp8,fp8,0,0.15089600086212157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,40,128,1,float16,float16,0,0.14994560480117797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,40,128,1,float16,fp8,0,0.1480512022972107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,40,128,1,fp8,fp8,0,0.14756640195846557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,1,128,1,float16,float16,0,0.07141600251197815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,1,128,1,float16,fp8,0,0.07040159702301026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,1,128,1,fp8,fp8,0,0.07063040137290955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,2,128,1,float16,float16,0,0.07516000270843506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,2,128,1,float16,fp8,0,0.06993280053138733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,2,128,1,fp8,fp8,0,0.07073760032653809
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,4,128,1,float16,float16,0,0.07886719703674316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,4,128,1,float16,fp8,0,0.07271040081977845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,4,128,1,fp8,fp8,0,0.07274399995803833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,8,128,1,float16,float16,0,0.08810719847679138
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,8,128,1,float16,fp8,0,0.08166720271110535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,8,128,1,fp8,fp8,0,0.08183839917182922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,1,128,1,float16,float16,0,3.540964889526367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,1,128,1,float16,fp8,0,3.4275630950927733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,1,128,1,fp8,fp8,0,3.440041732788086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,2,128,1,float16,float16,0,3.725199890136719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,2,128,1,float16,fp8,0,3.5856014251708985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,2,128,1,fp8,fp8,0,3.585433578491211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,4,128,1,float16,float16,0,4.016491317749024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,4,128,1,float16,fp8,0,3.913447952270508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,4,128,1,fp8,fp8,0,3.8624622344970705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,8,128,1,float16,float16,0,4.5759838104248045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,8,128,1,float16,fp8,0,4.475235366821289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,8,128,1,fp8,fp8,0,4.457035064697266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,40,128,1,float16,float16,0,4.79969596862793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,40,128,1,float16,fp8,0,4.560111999511719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,1,128,1,float16,float16,0,1.756488037109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,40,128,1,fp8,fp8,0,4.535662460327148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,1,128,1,float16,fp8,0,1.7207439422607422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,1,128,1,fp8,fp8,0,1.7155071258544923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,2,128,1,float16,float16,0,1.8115856170654296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,2,128,1,float16,fp8,0,1.8161855697631837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,2,128,1,fp8,fp8,0,1.7986448287963868
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,4,128,1,float16,float16,0,2.0200368881225588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,4,128,1,float16,fp8,0,1.9401439666748046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,4,128,1,fp8,fp8,0,1.9331136703491212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,8,128,1,float16,fp8,0,2.2405231475830076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,8,128,1,float16,float16,0,2.291646385192871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,1,128,1,float16,float16,0,0.8795951843261719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,8,128,1,fp8,fp8,0,2.378463935852051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,1,128,1,float16,fp8,0,0.8668848037719726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,40,128,1,float16,float16,0,2.415411186218262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,40,128,1,float16,fp8,0,2.2882591247558595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,1,128,1,fp8,fp8,0,0.8718111991882325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,2,128,1,float16,float16,0,0.9270784378051757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,40,128,1,fp8,fp8,0,2.3495647430419924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,2,128,1,float16,fp8,0,0.9032208442687988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,2,128,1,fp8,fp8,0,0.931328010559082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,4,128,1,float16,float16,0,0.9902031898498536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,4,128,1,float16,fp8,0,1.0003024101257325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,4,128,1,fp8,fp8,0,0.9877967834472656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,8,128,1,float16,float16,0,1.1496800422668456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,8,128,1,float16,fp8,0,1.145240020751953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,8,128,1,fp8,fp8,0,1.1384400367736816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,1,128,1,float16,float16,0,0.4624447822570801
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,1,128,1,float16,fp8,0,0.4506368160247803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,40,128,1,float16,float16,0,1.1922752380371093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,40,128,1,float16,fp8,0,1.160268783569336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,1,128,1,fp8,fp8,0,0.45787839889526366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,40,128,1,fp8,fp8,0,1.1463871955871583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,2,128,1,float16,float16,0,0.4828591823577881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,2,128,1,float16,fp8,0,0.4694943904876709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,2,128,1,fp8,fp8,0,0.47422399520874026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,4,128,1,float16,float16,0,0.5085855960845947
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,4,128,1,float16,fp8,0,0.5015071868896485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,4,128,1,fp8,fp8,0,0.506609582901001
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,8,128,1,float16,float16,0,0.600116777420044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,8,128,1,float16,fp8,0,0.5745296001434326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,8,128,1,fp8,fp8,0,0.5908271789550781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,40,128,1,float16,float16,0,0.6246543884277344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,40,128,1,float16,fp8,0,0.6018864154815674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,1,128,1,float16,float16,0,0.2386336088180542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,40,128,1,fp8,fp8,0,0.5891712188720704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,1,128,1,float16,fp8,0,0.23945279121398927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,1,128,1,fp8,fp8,0,0.24345760345458983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,2,128,1,float16,float16,0,0.2465008020401001
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,4,128,1,fp8,fp8,0,0.27023839950561523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,2,128,1,float16,fp8,0,0.2518320083618164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,2,128,1,fp8,fp8,0,0.25148959159851075
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,4,128,1,float16,float16,0,0.2732079982757568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,4,128,1,float16,fp8,0,0.264083194732666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,8,128,1,float16,float16,0,0.3065263986587524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,8,128,1,float16,fp8,0,0.3091536045074463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,8,128,1,fp8,fp8,0,0.29997758865356444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,40,128,1,float16,float16,0,0.327839994430542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,2,128,1,float16,float16,0,0.1417248010635376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,40,128,1,float16,fp8,0,0.31423680782318114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,1,128,1,float16,float16,0,0.134934401512146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,40,128,1,fp8,fp8,0,0.3170111894607544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,1,128,1,float16,fp8,0,0.13070080280303956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,1,128,1,fp8,fp8,0,0.13135839700698854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,2,128,1,float16,fp8,0,0.1416383981704712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,2,128,1,fp8,fp8,0,0.13984320163726807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,4,128,1,float16,float16,0,0.15008000135421753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,4,128,1,float16,fp8,0,0.14631680250167847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,4,128,1,fp8,fp8,0,0.14679839611053466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,8,128,1,float16,float16,0,0.16909120082855225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,8,128,1,float16,fp8,0,0.16592479944229127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,8,128,1,fp8,fp8,0,0.1673583984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,40,128,1,float16,float16,0,0.17912640571594238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,40,128,1,float16,fp8,0,0.17265280485153198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,40,128,1,fp8,fp8,0,0.17359839677810668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,1,128,1,float16,float16,0,0.07940639853477478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,1,128,1,float16,fp8,0,0.07513120174407958
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,1,128,1,fp8,fp8,0,0.07586719989776611
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,2,128,1,float16,float16,0,0.08124960064888001
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,2,128,1,float16,fp8,0,0.07709919810295104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,2,128,1,fp8,fp8,0,0.07620159983634948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,4,128,1,float16,float16,0,0.08727200031280517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,4,128,1,float16,fp8,0,0.08389599919319153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,4,128,1,fp8,fp8,0,0.08342559933662415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,8,128,1,float16,float16,0,0.09603360295295715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,8,128,1,float16,fp8,0,0.09451680183410645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,8,128,1,fp8,fp8,0,0.09354239702224731
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,40,128,1,float16,float16,0,0.10343040227890014
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,40,128,1,float16,fp8,0,0.09738879799842834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,40,128,1,fp8,fp8,0,0.09667999744415283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,1,128,1,float16,float16,0,0.04778720140457153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,2,128,1,fp8,fp8,0,0.047972801327705386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,1,128,1,float16,fp8,0,0.04753440022468567
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,1,128,1,fp8,fp8,0,0.04743359982967377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,2,128,1,float16,float16,0,0.04770239889621734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,2,128,1,float16,fp8,0,0.047310400009155276
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,4,128,1,float16,float16,0,0.048921599984169006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,4,128,1,float16,fp8,0,0.04937599897384644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,4,128,1,fp8,fp8,0,0.0485152006149292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,8,128,1,float16,float16,0,0.05538880228996277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,8,128,1,float16,fp8,0,0.05302879810333252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,8,128,1,fp8,fp8,0,0.05336959958076477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,1,128,1,float16,float16,0,2.534436798095703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,1,128,1,float16,fp8,0,2.5327999114990236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,1,128,1,fp8,fp8,0,2.5285263061523438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,2,128,1,float16,float16,0,2.7155279159545898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,2,128,1,float16,fp8,0,2.667148780822754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,2,128,1,fp8,fp8,0,2.6799087524414062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,4,128,1,float16,float16,0,3.0678768157958984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,4,128,1,float16,fp8,0,2.9655071258544923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,4,128,1,fp8,fp8,0,2.9701631546020506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,8,128,1,float16,float16,0,3.6516433715820313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,8,128,1,float16,fp8,0,3.579880142211914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,8,128,1,fp8,fp8,0,3.5794017791748045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,1,128,1,float16,float16,0,1.2657936096191407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,40,128,1,float16,float16,0,4.251252746582031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,1,128,1,float16,fp8,0,1.2795696258544922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,40,128,1,float16,fp8,0,4.123017501831055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,1,128,1,fp8,fp8,0,1.2611680030822754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,2,128,1,float16,float16,0,1.3462191581726075
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,2,128,1,float16,fp8,0,1.3272095680236817
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,2,128,1,fp8,fp8,0,1.3295295715332032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,40,128,1,fp8,fp8,0,4.088619232177734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,4,128,1,fp8,fp8,0,1.4972304344177245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,4,128,1,float16,fp8,0,1.5027536392211913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,4,128,1,float16,float16,0,1.5290559768676757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,8,128,1,float16,fp8,0,1.786123275756836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,8,128,1,float16,float16,0,1.830276870727539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,8,128,1,fp8,fp8,0,1.794046401977539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,40,128,1,float16,float16,0,2.10577278137207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,40,128,1,float16,fp8,0,2.0580368041992188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,1,128,1,fp8,fp8,0,0.6405087947845459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,1,128,1,float16,fp8,0,0.6499184131622314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,1,128,1,float16,float16,0,0.6481376171112061
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,2,128,1,float16,float16,0,0.6863264083862305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,2,128,1,fp8,fp8,0,0.6832399845123291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,2,128,1,float16,fp8,0,0.6787087917327881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,4,128,1,float16,float16,0,0.7667103767395019
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,4,128,1,float16,fp8,0,0.7524879932403564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,4,128,1,fp8,fp8,0,0.762499189376831
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,40,128,1,fp8,fp8,0,2.074665641784668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,8,128,1,float16,float16,0,0.9288592338562012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,8,128,1,float16,fp8,0,0.9090352058410645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,8,128,1,fp8,fp8,0,0.9121295928955078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,1,128,1,float16,float16,0,0.33660480976104734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,40,128,1,float16,float16,0,1.0677552223205566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,1,128,1,float16,fp8,0,0.3358448028564453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,1,128,1,fp8,fp8,0,0.3357327938079834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,40,128,1,float16,fp8,0,1.044718360900879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,40,128,1,fp8,fp8,0,1.0382752418518066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,2,128,1,float16,float16,0,0.35551040172576903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,2,128,1,float16,fp8,0,0.3544303894042969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,2,128,1,fp8,fp8,0,0.3543551921844482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,4,128,1,float16,float16,0,0.3938911914825439
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,4,128,1,float16,fp8,0,0.3919264078140259
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,4,128,1,fp8,fp8,0,0.3915776014328003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,8,128,1,float16,float16,0,0.4846191883087158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,8,128,1,float16,fp8,0,0.4637119770050049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,8,128,1,fp8,fp8,0,0.4651936054229736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,40,128,1,float16,float16,0,0.5504144191741943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,1,128,1,float16,float16,0,0.18098080158233643
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,40,128,1,float16,fp8,0,0.5378320217132568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,40,128,1,fp8,fp8,0,0.5467167854309082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,1,128,1,float16,fp8,0,0.1812175989151001
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,1,128,1,fp8,fp8,0,0.18494080305099486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,2,128,1,float16,float16,0,0.18959200382232666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,2,128,1,float16,fp8,0,0.18865439891815186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,2,128,1,fp8,fp8,0,0.18908640146255493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,4,128,1,float16,float16,0,0.2102799892425537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,4,128,1,float16,fp8,0,0.20806241035461426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,4,128,1,fp8,fp8,0,0.210809588432312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,8,128,1,float16,float16,0,0.2494976043701172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,8,128,1,float16,fp8,0,0.24762399196624757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,8,128,1,fp8,fp8,0,0.24557919502258302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,40,128,1,float16,float16,0,0.28763039112091066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,40,128,1,float16,fp8,0,0.2806639909744263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,1,128,1,float16,float16,0,0.10201120376586914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,2,128,1,fp8,fp8,0,0.10811840295791626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,40,128,1,fp8,fp8,0,0.2823551893234253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,1,128,1,float16,fp8,0,0.10326080322265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,1,128,1,fp8,fp8,0,0.10218240022659301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,8,128,1,float16,float16,0,0.13658080101013184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,2,128,1,float16,float16,0,0.10831040143966675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,2,128,1,float16,fp8,0,0.10820480585098266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,4,128,1,float16,float16,0,0.11837279796600342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,4,128,1,float16,fp8,0,0.1159168004989624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,4,128,1,fp8,fp8,0,0.11605119705200195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,8,128,1,float16,fp8,0,0.13708000183105468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,8,128,1,fp8,fp8,0,0.13561919927597046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,40,128,1,float16,float16,0,0.15783360004425048
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,40,128,1,float16,fp8,0,0.15518879890441895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,40,128,1,fp8,fp8,0,0.15375839471817015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,1,128,1,float16,float16,0,0.06475200057029724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,2,128,1,fp8,fp8,0,0.060782402753829956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,4,128,1,float16,float16,0,0.07019039988517761
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,1,128,1,float16,fp8,0,0.06050400137901306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,1,128,1,fp8,fp8,0,0.06099039912223816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,2,128,1,float16,float16,0,0.06499040126800537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,2,128,1,float16,fp8,0,0.06153759956359863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,4,128,1,float16,fp8,0,0.0699184000492096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,4,128,1,fp8,fp8,0,0.06730239987373351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,8,128,1,float16,float16,0,0.07865279912948608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,8,128,1,float16,fp8,0,0.07794719934463501
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,8,128,1,fp8,fp8,0,0.0792415976524353
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,40,128,1,float16,float16,0,0.09152799844741821
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,40,128,1,float16,fp8,0,0.0879584014415741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,40,128,1,fp8,fp8,0,0.08796799778938294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,1,128,1,float16,float16,0,0.03907040059566498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,1,128,1,float16,fp8,0,0.039847999811172485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,1,128,1,fp8,fp8,0,0.03967680037021637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,2,128,1,float16,float16,0,0.039524799585342406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,2,128,1,float16,fp8,0,0.039740800857543945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,2,128,1,fp8,fp8,0,0.04041439890861511
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,4,128,1,float16,float16,0,0.04073599874973297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,4,128,1,float16,fp8,0,0.04124639928340912
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,4,128,1,fp8,fp8,0,0.040715199708938596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,8,128,1,float16,float16,0,0.04653120040893555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,8,128,1,float16,fp8,0,0.04626719951629639
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,8,128,1,fp8,fp8,0,0.045337599515914914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,40,128,1,float16,float16,0,0.051990401744842527
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,40,128,1,float16,fp8,0,0.05135840177536011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,40,128,1,fp8,fp8,0,0.05088319778442383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,1,128,1,float16,float16,0,0.02961759865283966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,1,128,1,float16,fp8,0,0.030713599920272828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,1,128,1,fp8,fp8,0,0.03076159954071045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,2,128,1,float16,float16,0,0.02959359884262085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,2,128,1,float16,fp8,0,0.03091999888420105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,2,128,1,fp8,fp8,0,0.030904000997543334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,4,128,1,float16,float16,0,0.029879999160766602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,4,128,1,float16,fp8,0,0.031079998612403868
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,4,128,1,fp8,fp8,0,0.031147199869155883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,8,128,1,float16,float16,0,0.030828800797462464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,8,128,1,float16,fp8,0,0.03209919929504394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,1,128,1,float16,float16,0,1.0078960418701173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,8,128,1,fp8,fp8,0,0.03197920024394989
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,1,128,1,float16,fp8,0,1.0405887603759765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,1,128,1,fp8,fp8,0,1.0390159606933593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,2,128,1,float16,float16,0,1.073793601989746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,2,128,1,float16,fp8,0,1.1131343841552734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,2,128,1,fp8,fp8,0,1.120860767364502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,4,128,1,float16,float16,0,1.2385536193847657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,4,128,1,float16,fp8,0,1.2847040176391602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,4,128,1,fp8,fp8,0,1.2780511856079102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,8,128,1,float16,float16,0,1.5478832244873046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,1,128,1,float16,float16,0,0.5091152191162109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,8,128,1,float16,fp8,0,1.5600159645080567
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,1,128,1,float16,fp8,0,0.5373631954193115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,8,128,1,fp8,fp8,0,1.5630640029907226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,1,128,1,fp8,fp8,0,0.5337503910064697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,2,128,1,float16,float16,0,0.5473536014556885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,2,128,1,float16,fp8,0,0.5679632186889648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,2,128,1,fp8,fp8,0,0.5728367805480957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,40,128,1,float16,fp8,0,1.9641279220581054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,40,128,1,float16,float16,0,2.045587158203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,40,128,1,fp8,fp8,0,1.9472448348999023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,4,128,1,float16,float16,0,0.6274240016937256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,4,128,1,float16,fp8,0,0.6438543796539307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,4,128,1,fp8,fp8,0,0.6441440105438232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,8,128,1,float16,float16,0,0.7830527782440185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,8,128,1,float16,fp8,0,0.7887824058532715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,8,128,1,fp8,fp8,0,0.789788818359375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,1,128,1,float16,float16,0,0.26690239906311036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,1,128,1,float16,fp8,0,0.2783407926559448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,40,128,1,float16,float16,0,1.0313936233520509
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,40,128,1,float16,fp8,0,0.9861472129821778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,40,128,1,fp8,fp8,0,0.9826016426086426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,1,128,1,fp8,fp8,0,0.27890720367431643
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,2,128,1,float16,float16,0,0.2860624074935913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,2,128,1,fp8,fp8,0,0.29732480049133303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,2,128,1,float16,fp8,0,0.29736480712890623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,4,128,1,float16,float16,0,0.3292848110198975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,4,128,1,float16,fp8,0,0.33489439487457273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,4,128,1,fp8,fp8,0,0.33510560989379884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,8,128,1,float16,float16,0,0.4024208068847656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,8,128,1,float16,fp8,0,0.4048719882965088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,8,128,1,fp8,fp8,0,0.40528478622436526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,1,128,1,fp8,fp8,0,0.15281920433044432
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,40,128,1,float16,float16,0,0.5279327869415283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,1,128,1,float16,float16,0,0.14654239416122436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,40,128,1,float16,fp8,0,0.504260778427124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,40,128,1,fp8,fp8,0,0.511075210571289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,1,128,1,float16,fp8,0,0.1528880000114441
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,2,128,1,float16,float16,0,0.15476959943771362
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,2,128,1,float16,fp8,0,0.16007039546966553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,2,128,1,fp8,fp8,0,0.16055359840393066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,4,128,1,float16,float16,0,0.17439680099487304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,4,128,1,float16,fp8,0,0.17938400506973268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,4,128,1,fp8,fp8,0,0.17958400249481202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,8,128,1,float16,float16,0,0.21665120124816895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,8,128,1,float16,fp8,0,0.2153696060180664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,8,128,1,fp8,fp8,0,0.21453120708465576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,40,128,1,float16,float16,0,0.27689599990844727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,1,128,1,fp8,fp8,0,0.08780480027198792
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,40,128,1,float16,fp8,0,0.2639039993286133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,40,128,1,fp8,fp8,0,0.2639631986618042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,1,128,1,float16,float16,0,0.08399680256843567
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,1,128,1,float16,fp8,0,0.08835679888725281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,2,128,1,float16,float16,0,0.08988800048828124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,2,128,1,float16,fp8,0,0.0929199993610382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,2,128,1,fp8,fp8,0,0.09342880249023437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,4,128,1,float16,float16,0,0.09881600141525268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,4,128,1,float16,fp8,0,0.10173439979553223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,4,128,1,fp8,fp8,0,0.10104000568389893
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,8,128,1,float16,float16,0,0.11999200582504273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,8,128,1,float16,fp8,0,0.11952639818191528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,8,128,1,fp8,fp8,0,0.119651198387146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,40,128,1,float16,float16,0,0.1517807960510254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,40,128,1,float16,fp8,0,0.1446671962738037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,40,128,1,fp8,fp8,0,0.14457759857177735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,1,128,1,float16,float16,0,0.0551967978477478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,1,128,1,float16,fp8,0,0.05345439910888672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,1,128,1,fp8,fp8,0,0.05254719853401184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,2,128,1,float16,float16,0,0.05584319829940796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,2,128,1,float16,fp8,0,0.05433599948883057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,2,128,1,fp8,fp8,0,0.05404319763183594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,4,128,1,float16,float16,0,0.06232799887657166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,4,128,1,float16,fp8,0,0.061921602487564086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,4,128,1,fp8,fp8,0,0.0609279990196228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,8,128,1,float16,float16,0,0.07072319984436035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,40,128,1,fp8,fp8,0,0.08116480112075805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,8,128,1,float16,fp8,0,0.07036319971084595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,8,128,1,fp8,fp8,0,0.07035520076751708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,40,128,1,float16,float16,0,0.08751360177993775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,40,128,1,float16,fp8,0,0.08037760257720947
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,1,128,1,float16,float16,0,0.03446879982948303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,1,128,1,float16,fp8,0,0.03597280085086822
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,1,128,1,fp8,fp8,0,0.03597759902477264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,2,128,1,float16,float16,0,0.034302398562431335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,2,128,1,float16,fp8,0,0.036268800497055054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,2,128,1,fp8,fp8,0,0.036375999450683594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,4,128,1,float16,float16,0,0.03553920090198517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,40,128,1,float16,float16,0,0.0488400012254715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,4,128,1,float16,fp8,0,0.0373744010925293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,4,128,1,fp8,fp8,0,0.03701280057430267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,8,128,1,float16,float16,0,0.041920000314712526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,8,128,1,float16,fp8,0,0.04159359931945801
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,8,128,1,fp8,fp8,0,0.041457599401474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,40,128,1,float16,fp8,0,0.04751839935779571
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,40,128,1,fp8,fp8,0,0.04720160067081451
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,1,128,1,float16,float16,0,0.027878400683403016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,1,128,1,float16,fp8,0,0.02895680069923401
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,1,128,1,fp8,fp8,0,0.028911998867988585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,2,128,1,float16,float16,0,0.027911999821662904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,2,128,1,float16,fp8,0,0.029067200422286988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,2,128,1,fp8,fp8,0,0.028880000114440918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,4,128,1,float16,float16,0,0.028305599093437196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,4,128,1,float16,fp8,0,0.029396799206733704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,4,128,1,fp8,fp8,0,0.02922079861164093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,8,128,1,float16,float16,0,0.029126399755477907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,8,128,1,float16,fp8,0,0.030035200715065002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,8,128,1,fp8,fp8,0,0.030616000294685364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,40,128,1,float16,float16,0,0.030862399935722352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,40,128,1,float16,fp8,0,0.031431999802589414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,40,128,1,fp8,fp8,0,0.0314736008644104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,1,128,1,float16,float16,0,0.023761600255966187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,1,128,1,float16,fp8,0,0.024806399643421174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,1,128,1,fp8,fp8,0,0.024883200228214265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,2,128,1,float16,float16,0,0.02399040013551712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,2,128,1,float16,fp8,0,0.025143998861312866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,2,128,1,fp8,fp8,0,0.024553599953651428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,4,128,1,float16,float16,0,0.024060800671577454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,4,128,1,float16,fp8,0,0.024843199551105498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,4,128,1,fp8,fp8,0,0.024774399399757386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,8,128,1,float16,float16,0,0.02466239929199219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,8,128,1,float16,fp8,0,0.025088000297546386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,8,128,1,fp8,fp8,0,0.024884800612926482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,1,128,1,float16,float16,0,0.48990721702575685
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,1,128,1,float16,fp8,0,0.5281392097473144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,1,128,1,fp8,fp8,0,0.5289055824279785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,2,128,1,float16,float16,0,0.5274879932403564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,2,128,1,float16,fp8,0,0.5692912101745605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,2,128,1,fp8,fp8,0,0.5707920074462891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,4,128,1,float16,float16,0,0.6060800075531005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,4,128,1,float16,fp8,0,0.6491248130798339
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,8,128,1,float16,float16,0,0.7513792037963867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,4,128,1,fp8,fp8,0,0.6486591815948486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,8,128,1,float16,fp8,0,0.8077648162841797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,8,128,1,fp8,fp8,0,0.8082207679748535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,1,128,1,float16,float16,0,0.25759999752044677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,1,128,1,float16,fp8,0,0.27672638893127444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,1,128,1,fp8,fp8,0,0.27620959281921387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,40,128,1,float16,float16,0,1.0313632011413574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,2,128,1,float16,float16,0,0.27566239833831785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,40,128,1,float16,fp8,0,1.0044159889221191
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,2,128,1,float16,fp8,0,0.2974303960800171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,40,128,1,fp8,fp8,0,1.006056022644043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,2,128,1,fp8,fp8,0,0.29769918918609617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,4,128,1,float16,float16,0,0.3117743968963623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,8,128,1,float16,fp8,0,0.41309280395507814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,4,128,1,float16,fp8,0,0.33600640296936035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,4,128,1,fp8,fp8,0,0.33541600704193114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,8,128,1,float16,float16,0,0.38744320869445803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,8,128,1,fp8,fp8,0,0.4122640132904053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,40,128,1,float16,float16,0,0.5288320064544678
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,40,128,1,float16,fp8,0,0.510640001296997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,1,128,1,float16,float16,0,0.14005119800567628
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,40,128,1,fp8,fp8,0,0.5147391796112061
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,1,128,1,float16,fp8,0,0.15232319831848146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,1,128,1,fp8,fp8,0,0.1525231957435608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,2,128,1,float16,float16,0,0.14838080406188964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,2,128,1,float16,fp8,0,0.16131360530853273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,2,128,1,fp8,fp8,0,0.1616752028465271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,4,128,1,float16,float16,0,0.16619999408721925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,4,128,1,float16,fp8,0,0.18071520328521729
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,4,128,1,fp8,fp8,0,0.18133759498596191
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,8,128,1,float16,float16,0,0.20337600708007814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,8,128,1,float16,fp8,0,0.21763360500335693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,8,128,1,fp8,fp8,0,0.2187567949295044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,40,128,1,float16,float16,0,0.27706079483032225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,40,128,1,float16,fp8,0,0.2639103889465332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,40,128,1,fp8,fp8,0,0.26630239486694335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,1,128,1,float16,float16,0,0.08294879794120788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,1,128,1,float16,fp8,0,0.08567039966583252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,4,128,1,float16,float16,0,0.09595040082931519
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,1,128,1,fp8,fp8,0,0.08548960089683533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,2,128,1,float16,float16,0,0.08820639848709107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,2,128,1,float16,fp8,0,0.09229120016098022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,2,128,1,fp8,fp8,0,0.09177600145339966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,4,128,1,float16,fp8,0,0.09963679909706116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,4,128,1,fp8,fp8,0,0.09982720017433167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,8,128,1,float16,float16,0,0.11454559564590454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,8,128,1,float16,fp8,0,0.11840319633483887
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,8,128,1,fp8,fp8,0,0.1187391996383667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,40,128,1,float16,float16,0,0.15094399452209473
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,40,128,1,float16,fp8,0,0.14164639711380006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,40,128,1,fp8,fp8,0,0.14130719900131225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,1,128,1,float16,float16,0,0.052350401878356934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,1,128,1,float16,fp8,0,0.04925599992275238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,1,128,1,fp8,fp8,0,0.047147199511528015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,2,128,1,float16,float16,0,0.053225600719451906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,2,128,1,float16,fp8,0,0.05079519748687744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,2,128,1,fp8,fp8,0,0.050302398204803464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,4,128,1,float16,float16,0,0.05948479771614075
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,4,128,1,float16,fp8,0,0.05504159927368164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,4,128,1,fp8,fp8,0,0.057094401121139525
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,8,128,1,float16,float16,0,0.06793760061264038
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,8,128,1,float16,fp8,0,0.06664159893989563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,8,128,1,fp8,fp8,0,0.06584640145301819
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,40,128,1,float16,float16,0,0.08585280179977417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,40,128,1,float16,fp8,0,0.07839040160179138
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,40,128,1,fp8,fp8,0,0.07779840230941773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,1,128,1,float16,float16,0,0.03296639919281006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,1,128,1,float16,fp8,0,0.034590399265289305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,1,128,1,fp8,fp8,0,0.03454560041427612
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,2,128,1,float16,float16,0,0.03309119939804077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,2,128,1,float16,fp8,0,0.03490239977836609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,2,128,1,fp8,fp8,0,0.034995201230049136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,4,128,1,float16,fp8,0,0.0355648010969162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,4,128,1,float16,float16,0,0.03440000116825104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,4,128,1,fp8,fp8,0,0.03577919900417328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,8,128,1,float16,float16,0,0.040345600247383116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,8,128,1,float16,fp8,0,0.040296000242233274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,8,128,1,fp8,fp8,0,0.039950400590896606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,40,128,1,float16,float16,0,0.04787839949131012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,40,128,1,float16,fp8,0,0.04511359930038452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,40,128,1,fp8,fp8,0,0.04507679939270019
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,1,128,1,float16,float16,0,0.027795198559761047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,1,128,1,float16,fp8,0,0.02925119996070862
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,1,128,1,fp8,fp8,0,0.029028800129890443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,2,128,1,float16,float16,0,0.02805759906768799
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,2,128,1,float16,fp8,0,0.028992000222206115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,2,128,1,fp8,fp8,0,0.029238399863243104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,4,128,1,float16,float16,0,0.028152000904083253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,4,128,1,float16,fp8,0,0.029177600145339967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,4,128,1,fp8,fp8,0,0.029097598791122437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,8,128,1,float16,float16,0,0.029161599278450013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,8,128,1,float16,fp8,0,0.030160000920295714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,8,128,1,fp8,fp8,0,0.03033599853515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,40,128,1,float16,float16,0,0.02958720028400421
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,40,128,1,float16,fp8,0,0.029927998781204224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,40,128,1,fp8,fp8,0,0.029867199063301087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,1,128,1,float16,float16,0,0.02222079932689667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,1,128,1,float16,fp8,0,0.02297919988632202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,1,128,1,fp8,fp8,0,0.023369599878787995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,2,128,1,float16,float16,0,0.022494399547576906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,2,128,1,float16,fp8,0,0.02316959947347641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,2,128,1,fp8,fp8,0,0.023358400166034698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,4,128,1,float16,float16,0,0.02261279970407486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,4,128,1,float16,fp8,0,0.023395200073719025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,4,128,1,fp8,fp8,0,0.02300799936056137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,8,128,1,float16,float16,0,0.022896000742912294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,8,128,1,float16,fp8,0,0.023524799942970277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,8,128,1,fp8,fp8,0,0.023228800296783446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,40,128,1,float16,float16,0,0.024329599738121033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,40,128,1,float16,fp8,0,0.024289600551128387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,40,128,1,fp8,fp8,0,0.02473919987678528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,1,128,1,float16,float16,0,0.02141920030117035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,1,128,1,float16,fp8,0,0.021831999719142913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,1,128,1,fp8,fp8,0,0.021848000586032867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,2,128,1,float16,float16,0,0.021499200165271758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,2,128,1,float16,fp8,0,0.02218399941921234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,2,128,1,fp8,fp8,0,0.022256000339984892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,4,128,1,float16,float16,0,0.021547199785709382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,4,128,1,float16,fp8,0,0.022499200701713563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,4,128,1,fp8,fp8,0,0.022116799652576447
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,8,128,1,float16,float16,0,0.02170879989862442
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,8,128,1,float16,fp8,0,0.022436800599098205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,8,128,1,fp8,fp8,0,0.022243200242519377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,1,128,1,float16,float16,0,0.26147520542144775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,1,128,1,float16,fp8,0,0.27746560573577883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,1,128,1,fp8,fp8,0,0.278004789352417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,2,128,1,float16,float16,0,0.2762655973434448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,2,128,1,float16,fp8,0,0.30032958984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,2,128,1,fp8,fp8,0,0.2986464023590088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,4,128,1,float16,float16,0,0.3137264013290405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,4,128,1,float16,fp8,0,0.3373823881149292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,4,128,1,fp8,fp8,0,0.33654079437255857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,8,128,1,float16,float16,0,0.41905918121337893
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,8,128,1,float16,fp8,0,0.4427584171295166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,8,128,1,fp8,fp8,0,0.44495358467102053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,40,128,1,float16,float16,0,0.585368013381958
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,1,128,1,float16,float16,0,0.1424015998840332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,1,128,1,float16,fp8,0,0.15259360074996947
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,40,128,1,float16,fp8,0,0.5982367992401123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,40,128,1,fp8,fp8,0,0.5970160007476807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,1,128,1,fp8,fp8,0,0.15324800014495848
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,2,128,1,float16,float16,0,0.15088000297546386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,2,128,1,float16,fp8,0,0.16223039627075195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,2,128,1,fp8,fp8,0,0.1616960048675537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,4,128,1,float16,float16,0,0.16959840059280396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,4,128,1,float16,fp8,0,0.18066240549087526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,4,128,1,fp8,fp8,0,0.18067519664764403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,8,128,1,float16,float16,0,0.22146880626678467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,8,128,1,float16,fp8,0,0.23477280139923096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,8,128,1,fp8,fp8,0,0.23315200805664063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,40,128,1,float16,float16,0,0.3060208082199097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,40,128,1,float16,fp8,0,0.3098112106323242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,1,128,1,float16,float16,0,0.08259199857711792
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,40,128,1,fp8,fp8,0,0.3116080045700073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,1,128,1,float16,fp8,0,0.08677120208740234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,1,128,1,fp8,fp8,0,0.08702880144119263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,2,128,1,float16,float16,0,0.08744159936904908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,2,128,1,float16,fp8,0,0.09348480105400085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,8,128,1,float16,fp8,0,0.1267024040222168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,2,128,1,fp8,fp8,0,0.09314560294151306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,4,128,1,float16,float16,0,0.09781280159950256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,4,128,1,float16,fp8,0,0.10044000148773194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,4,128,1,fp8,fp8,0,0.10047999620437623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,8,128,1,float16,float16,0,0.12296960353851319
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,8,128,1,fp8,fp8,0,0.12695839405059814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,1,128,1,fp8,fp8,0,0.04768480062484741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,1,128,1,float16,fp8,0,0.04789440035820007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,40,128,1,float16,float16,0,0.16537599563598632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,1,128,1,float16,float16,0,0.05251680016517639
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,2,128,1,float16,float16,0,0.05329279899597168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,2,128,1,float16,fp8,0,0.04996800124645233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,40,128,1,float16,fp8,0,0.16365760564804077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,40,128,1,fp8,fp8,0,0.16496800184249877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,2,128,1,fp8,fp8,0,0.049963200092315675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,4,128,1,float16,float16,0,0.05929440259933472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,4,128,1,float16,fp8,0,0.05769280195236206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,40,128,1,float16,fp8,0,0.08532800078392029
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,40,128,1,fp8,fp8,0,0.08593599796295166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,4,128,1,fp8,fp8,0,0.057067197561264035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,8,128,1,float16,float16,0,0.0716048002243042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,8,128,1,float16,fp8,0,0.07023680210113525
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,8,128,1,fp8,fp8,0,0.07034239768981934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,40,128,1,float16,float16,0,0.09312480092048644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,1,128,1,float16,fp8,0,0.03468480110168457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,1,128,1,float16,float16,0,0.0327455997467041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,1,128,1,fp8,fp8,0,0.03471519947052002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,2,128,1,float16,float16,0,0.032864001393318173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,2,128,1,float16,fp8,0,0.03446399867534637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,2,128,1,fp8,fp8,0,0.03474720120429993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,4,128,1,float16,float16,0,0.03451679944992066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,4,128,1,float16,fp8,0,0.03569439947605133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,4,128,1,fp8,fp8,0,0.03562400043010712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,8,128,1,float16,float16,0,0.04023680090904236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,8,128,1,float16,fp8,0,0.03975839912891388
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,8,128,1,fp8,fp8,0,0.03986240029335022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,40,128,1,float16,float16,0,0.05118079781532288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,40,128,1,float16,fp8,0,0.04919039905071258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,40,128,1,fp8,fp8,0,0.049403199553489686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,1,128,1,float16,float16,0,0.02772960066795349
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,1,128,1,float16,fp8,0,0.029028800129890443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,1,128,1,fp8,fp8,0,0.029500800371170043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,2,128,1,float16,float16,0,0.027851200103759764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,2,128,1,float16,fp8,0,0.029054400324821473
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,2,128,1,fp8,fp8,0,0.029212799668312073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,4,128,1,float16,float16,0,0.028043198585510253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,4,128,1,float16,fp8,0,0.02921920120716095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,4,128,1,fp8,fp8,0,0.02943359911441803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,1,128,1,float16,float16,0,0.022204799950122832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,8,128,1,float16,float16,0,0.02871200144290924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,8,128,1,float16,fp8,0,0.030272001028060914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,8,128,1,fp8,fp8,0,0.030137598514556885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,40,128,1,float16,float16,0,0.032318401336669925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,40,128,1,float16,fp8,0,0.03372159898281098
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,40,128,1,fp8,fp8,0,0.03364000022411347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,1,128,1,float16,fp8,0,0.022991999983787537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,1,128,1,fp8,fp8,0,0.02314720004796982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,2,128,1,float16,float16,0,0.02229759991168976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,2,128,1,float16,fp8,0,0.02338559925556183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,2,128,1,fp8,fp8,0,0.02330079972743988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,4,128,1,float16,float16,0,0.02266560047864914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,4,128,1,float16,fp8,0,0.022937600314617158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,4,128,1,fp8,fp8,0,0.02316800057888031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,8,128,1,float16,float16,0,0.022651199996471406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,8,128,1,float16,fp8,0,0.022841599583625794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,8,128,1,fp8,fp8,0,0.02343039959669113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,40,128,1,float16,float16,0,0.023332799971103668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,40,128,1,float16,fp8,0,0.024051199853420257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,40,128,1,fp8,fp8,0,0.02407039999961853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,1,128,1,float16,float16,0,0.021433599293231964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,1,128,1,float16,fp8,0,0.022283199429512023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,1,128,1,fp8,fp8,0,0.021876800060272216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,2,128,1,float16,float16,0,0.021436800062656403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,2,128,1,float16,fp8,0,0.022555199265480042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,2,128,1,fp8,fp8,0,0.02208160012960434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,4,128,1,float16,float16,0,0.021516799926757812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,4,128,1,float16,fp8,0,0.02224159985780716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,4,128,1,fp8,fp8,0,0.022268800437450408
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,8,128,1,float16,float16,0,0.02152000069618225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,8,128,1,float16,fp8,0,0.022614400088787078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,8,128,1,fp8,fp8,0,0.022407999634742735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,40,128,1,float16,float16,0,0.021452799439430237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,40,128,1,float16,fp8,0,0.02215680032968521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,40,128,1,fp8,fp8,0,0.022019200026988983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,1,128,1,float16,float16,0,0.02088160067796707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,1,128,1,float16,fp8,0,0.021639999747276307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,1,128,1,fp8,fp8,0,0.02162880003452301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,2,128,1,float16,float16,0,0.020900799334049223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,2,128,1,float16,fp8,0,0.0217631995677948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,2,128,1,fp8,fp8,0,0.021649600565433504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,4,128,1,float16,float16,0,0.021076799929142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,4,128,1,float16,fp8,0,0.021515199542045595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,4,128,1,fp8,fp8,0,0.0216511994600296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,8,128,1,float16,float16,0,0.021247999370098115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,8,128,1,float16,fp8,0,0.022071999311447144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,8,128,1,fp8,fp8,0,0.021891200542449953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,1,128,1,float16,float16,0,0.14272160530090333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,1,128,1,float16,fp8,0,0.1539199948310852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,1,128,1,fp8,fp8,0,0.15492160320281984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,2,128,1,float16,float16,0,0.1501039981842041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,4,128,1,fp8,fp8,0,0.19614399671554567
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,8,128,1,float16,float16,0,0.24363999366760253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,2,128,1,float16,fp8,0,0.16257760524749756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,2,128,1,fp8,fp8,0,0.1631343960762024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,4,128,1,float16,float16,0,0.1838719964027405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,4,128,1,float16,fp8,0,0.19717119932174682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,8,128,1,float16,fp8,0,0.2595376014709473
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,8,128,1,fp8,fp8,0,0.2585520029067993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,40,128,1,float16,float16,0,0.37947518825531007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,1,128,1,float16,float16,0,0.0819487988948822
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,40,128,1,float16,fp8,0,0.39435520172119143
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,40,128,1,fp8,fp8,0,0.39474079608917234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,1,128,1,float16,fp8,0,0.08770880103111267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,1,128,1,fp8,fp8,0,0.08830879926681519
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,2,128,1,float16,float16,0,0.08944799900054931
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,2,128,1,float16,fp8,0,0.09305920004844666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,2,128,1,fp8,fp8,0,0.09397280216217041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,4,128,1,float16,float16,0,0.10459040403366089
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,4,128,1,float16,fp8,0,0.10839840173721313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,4,128,1,fp8,fp8,0,0.10853439569473267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,8,128,1,float16,float16,0,0.1340224027633667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,8,128,1,float16,fp8,0,0.14365919828414916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,8,128,1,fp8,fp8,0,0.14344639778137208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,40,128,1,float16,float16,0,0.20151998996734619
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,40,128,1,float16,fp8,0,0.20975840091705322
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,1,128,1,float16,float16,0,0.05326560139656067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,40,128,1,fp8,fp8,0,0.21085119247436523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,1,128,1,float16,fp8,0,0.049384000897407535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,1,128,1,fp8,fp8,0,0.05011839866638183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,2,128,1,float16,float16,0,0.05486239790916443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,2,128,1,float16,fp8,0,0.049595201015472413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,2,128,1,fp8,fp8,0,0.049595201015472413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,4,128,1,float16,float16,0,0.06424480080604553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,4,128,1,float16,fp8,0,0.06120479702949524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,4,128,1,fp8,fp8,0,0.06217120289802551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,8,128,1,float16,float16,0,0.07954239845275879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,8,128,1,float16,fp8,0,0.08039839863777161
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,8,128,1,fp8,fp8,0,0.08048480153083801
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,40,128,1,float16,float16,0,0.11136640310287475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,40,128,1,float16,fp8,0,0.10943200588226318
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,40,128,1,fp8,fp8,0,0.10964640378952026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,1,128,1,float16,float16,0,0.033000001311302186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,1,128,1,float16,fp8,0,0.03499200046062469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,1,128,1,fp8,fp8,0,0.03490560054779053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,2,128,1,float16,float16,0,0.03305279910564422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,2,128,1,float16,fp8,0,0.035174399614334106
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,2,128,1,fp8,fp8,0,0.03516159951686859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,4,128,1,float16,float16,0,0.03377760052680969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,4,128,1,float16,fp8,0,0.035806399583816526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,4,128,1,fp8,fp8,0,0.03569599986076355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,8,128,1,float16,float16,0,0.043303999304771426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,8,128,1,float16,fp8,0,0.044519999623298646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,8,128,1,fp8,fp8,0,0.04467839896678925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,40,128,1,float16,float16,0,0.058417600393295285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,40,128,1,float16,fp8,0,0.05826879739761352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,40,128,1,fp8,fp8,0,0.05834400057792664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,1,128,1,float16,float16,0,0.02794399857521057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,1,128,1,float16,fp8,0,0.028960001468658448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,1,128,1,fp8,fp8,0,0.029547199606895447
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,2,128,1,float16,float16,0,0.02744159996509552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,2,128,1,float16,fp8,0,0.029121598601341246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,2,128,1,fp8,fp8,0,0.028964799642562867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,4,128,1,float16,float16,0,0.02792479991912842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,4,128,1,float16,fp8,0,0.029398399591445922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,4,128,1,fp8,fp8,0,0.029131200909614564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,8,128,1,float16,float16,0,0.02852959930896759
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,8,128,1,float16,fp8,0,0.030063998699188233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,8,128,1,fp8,fp8,0,0.030033600330352784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,40,128,1,float16,float16,0,0.03618240058422088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,40,128,1,float16,fp8,0,0.038166400790214536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,40,128,1,fp8,fp8,0,0.0382095992565155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,1,128,1,float16,float16,0,0.02211039960384369
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,1,128,1,float16,fp8,0,0.02313919961452484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,1,128,1,fp8,fp8,0,0.022963200509548188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,2,128,1,float16,float16,0,0.022300800681114195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,2,128,1,float16,fp8,0,0.023022399842739107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,2,128,1,fp8,fp8,0,0.023156799376010895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,4,128,1,float16,float16,0,0.02250239998102188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,4,128,1,float16,fp8,0,0.02295520007610321
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,4,128,1,fp8,fp8,0,0.02332639992237091
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,8,128,1,float16,float16,0,0.022388799488544463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,8,128,1,float16,fp8,0,0.0233487993478775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,8,128,1,fp8,fp8,0,0.023201599717140198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,40,128,1,float16,float16,0,0.026926401257514953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,40,128,1,float16,fp8,0,0.028068798780441283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,40,128,1,fp8,fp8,0,0.028240001201629637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,1,128,1,float16,float16,0,0.02119999974966049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,1,128,1,float16,fp8,0,0.02215680032968521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,1,128,1,fp8,fp8,0,0.022169600427150726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,2,128,1,float16,float16,0,0.02130880057811737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,2,128,1,float16,fp8,0,0.02206239998340607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,2,128,1,fp8,fp8,0,0.022337600588798523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,4,128,1,float16,float16,0,0.02160159945487976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,4,128,1,float16,fp8,0,0.02221439927816391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,4,128,1,fp8,fp8,0,0.02236959934234619
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,8,128,1,float16,float16,0,0.02138399928808212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,8,128,1,float16,fp8,0,0.022191999852657317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,1,128,1,fp8,fp8,0,0.0217616006731987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,8,128,1,fp8,fp8,0,0.02229759991168976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,40,128,1,float16,float16,0,0.021483199298381807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,40,128,1,float16,fp8,0,0.022260800004005432
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,40,128,1,fp8,fp8,0,0.0221903994679451
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,1,128,1,float16,float16,0,0.02085600048303604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,1,128,1,float16,fp8,0,0.021667200326919555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,2,128,1,float16,float16,0,0.02095839977264404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,2,128,1,float16,fp8,0,0.021550400555133818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,2,128,1,fp8,fp8,0,0.021695999801158904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,40,128,1,float16,float16,0,0.020977599918842314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,4,128,1,float16,float16,0,0.02088640034198761
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,4,128,1,float16,fp8,0,0.021649600565433504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,4,128,1,fp8,fp8,0,0.021779200434684752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,8,128,1,float16,float16,0,0.02136159986257553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,8,128,1,float16,fp8,0,0.022308799624443054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,8,128,1,fp8,fp8,0,0.02203039973974228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,40,128,1,float16,fp8,0,0.021796800196170807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,40,128,1,fp8,fp8,0,0.021726399660110474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,1,128,1,float16,float16,0,0.020526400208473204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,1,128,1,float16,fp8,0,0.021254399418830873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,1,128,1,fp8,fp8,0,0.021324799954891206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,2,128,1,float16,float16,0,0.020598399639129638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,2,128,1,float16,fp8,0,0.021558399498462676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,2,128,1,fp8,fp8,0,0.021272000670433045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,4,128,1,float16,float16,0,0.02067680060863495
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,4,128,1,float16,fp8,0,0.021379199624061585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,4,128,1,fp8,fp8,0,0.021388800442218782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,8,128,1,float16,float16,0,0.020766399800777435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,8,128,1,float16,fp8,0,0.02144159972667694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,8,128,1,fp8,fp8,0,0.021648000180721282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,1,128,1,float16,float16,0,0.08425279855728149
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,1,128,1,float16,fp8,0,0.08945599794387818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,1,128,1,fp8,fp8,0,0.0885807991027832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,2,128,1,float16,float16,0,0.09585599899291992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,8,128,1,float16,float16,0,0.13880800008773803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,2,128,1,float16,fp8,0,0.10227199792861938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,2,128,1,fp8,fp8,0,0.10157120227813721
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,4,128,1,float16,float16,0,0.11521279811859131
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,4,128,1,float16,fp8,0,0.12592639923095703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,4,128,1,fp8,fp8,0,0.12596800327301025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,8,128,1,float16,fp8,0,0.1463055968284607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,8,128,1,fp8,fp8,0,0.14648319482803346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,40,128,1,float16,float16,0,0.27764480113983153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,40,128,1,float16,fp8,0,0.2995248079299927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,1,128,1,float16,float16,0,0.05367839932441711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,40,128,1,fp8,fp8,0,0.3001951932907104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,1,128,1,float16,fp8,0,0.05006719827651977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,1,128,1,fp8,fp8,0,0.050704002380371094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,2,128,1,float16,float16,0,0.05955680012702942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,2,128,1,float16,fp8,0,0.05613440275192261
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,2,128,1,fp8,fp8,0,0.05488640069961548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,4,128,1,float16,float16,0,0.07180799841880799
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,4,128,1,float16,fp8,0,0.07018240094184876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,4,128,1,fp8,fp8,0,0.06970239877700805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,8,128,1,float16,float16,0,0.0813968002796173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,8,128,1,float16,fp8,0,0.08214560151100159
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,8,128,1,fp8,fp8,0,0.08252639770507812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,40,128,1,float16,float16,0,0.15049279928207399
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,40,128,1,float16,fp8,0,0.15686240196228027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,1,128,1,float16,float16,0,0.033430400490760806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,40,128,1,fp8,fp8,0,0.15616799592971803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,1,128,1,float16,fp8,0,0.03528000116348266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,1,128,1,fp8,fp8,0,0.0355648010969162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,2,128,1,float16,float16,0,0.03300800025463104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,4,128,1,fp8,fp8,0,0.04048640131950378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,2,128,1,float16,fp8,0,0.03565919995307922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,2,128,1,fp8,fp8,0,0.03537600040435791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,4,128,1,float16,float16,0,0.03759520053863526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,4,128,1,float16,fp8,0,0.040403199195861814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,8,128,1,float16,float16,0,0.04349919855594635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,8,128,1,float16,fp8,0,0.04536960124969482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,8,128,1,fp8,fp8,0,0.044654399156570435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,40,128,1,float16,float16,0,0.07752959728240967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,40,128,1,float16,fp8,0,0.08133599758148194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,40,128,1,fp8,fp8,0,0.08127040266990662
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,1,128,1,float16,float16,0,0.028071999549865723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,1,128,1,float16,fp8,0,0.029635199904441835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,1,128,1,fp8,fp8,0,0.029528000950813295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,2,128,1,float16,float16,0,0.027937600016593934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,2,128,1,float16,fp8,0,0.029491201043128967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,2,128,1,fp8,fp8,0,0.029411199688911437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,4,128,1,float16,float16,0,0.028281599283218384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,4,128,1,float16,fp8,0,0.0295087993144989
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,4,128,1,fp8,fp8,0,0.029684799909591674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,8,128,1,float16,float16,0,0.029022398591041564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,1,128,1,float16,float16,0,0.02232159972190857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,8,128,1,float16,fp8,0,0.03003840148448944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,8,128,1,fp8,fp8,0,0.03030720055103302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,40,128,1,float16,float16,0,0.04377920031547546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,40,128,1,float16,fp8,0,0.04746240079402923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,40,128,1,fp8,fp8,0,0.04726240038871765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,1,128,1,float16,fp8,0,0.023188799619674683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,1,128,1,fp8,fp8,0,0.02314720004796982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,2,128,1,float16,float16,0,0.022278399765491487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,2,128,1,float16,fp8,0,0.02338559925556183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,2,128,1,fp8,fp8,0,0.023043200373649597
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,4,128,1,float16,float16,0,0.02231840044260025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,4,128,1,float16,fp8,0,0.02297919988632202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,4,128,1,fp8,fp8,0,0.023196800053119658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,8,128,1,float16,float16,0,0.022543999552726745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,8,128,1,float16,fp8,0,0.02327200025320053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,8,128,1,fp8,fp8,0,0.023209600150585173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,40,128,1,float16,float16,0,0.03083199858665466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,40,128,1,float16,fp8,0,0.03297280073165894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,40,128,1,fp8,fp8,0,0.03272959887981415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,1,128,1,float16,float16,0,0.021742400527000428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,1,128,1,float16,fp8,0,0.022204799950122832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,1,128,1,fp8,fp8,0,0.022336000204086305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,8,128,1,float16,float16,0,0.021265600621700288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,2,128,1,float16,float16,0,0.02149759978055954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,2,128,1,float16,fp8,0,0.022180800139904023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,2,128,1,fp8,fp8,0,0.02237440049648285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,4,128,1,float16,float16,0,0.02160640060901642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,4,128,1,float16,fp8,0,0.02223680019378662
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,4,128,1,fp8,fp8,0,0.022360000014305114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,8,128,1,float16,fp8,0,0.022201600670814513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,8,128,1,fp8,fp8,0,0.022368000447750093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,40,128,1,float16,float16,0,0.02486719936132431
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,40,128,1,float16,fp8,0,0.026263999938964843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,40,128,1,fp8,fp8,0,0.026553601026535034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,1,128,1,float16,float16,0,0.020795199275016784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,1,128,1,float16,fp8,0,0.021491199731826782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,1,128,1,fp8,fp8,0,0.021695999801158904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,2,128,1,float16,float16,0,0.020815999805927278
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,2,128,1,float16,fp8,0,0.021686400473117828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,2,128,1,fp8,fp8,0,0.02157920002937317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,4,128,1,float16,float16,0,0.021134400367736818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,4,128,1,float16,fp8,0,0.022126400470733644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,4,128,1,fp8,fp8,0,0.021771200001239777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,1,128,1,float16,float16,0,0.020694400370121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,8,128,1,float16,float16,0,0.02083200067281723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,8,128,1,float16,fp8,0,0.021793599426746368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,8,128,1,fp8,fp8,0,0.021615999937057494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,40,128,1,float16,float16,0,0.021167999505996703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,40,128,1,float16,fp8,0,0.021716800332069398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,40,128,1,fp8,fp8,0,0.02173279970884323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,1,128,1,float16,fp8,0,0.02136480063199997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,1,128,1,fp8,fp8,0,0.021348799765110015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,2,128,1,float16,float16,0,0.02048480063676834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,2,128,1,float16,fp8,0,0.02129279971122742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,2,128,1,fp8,fp8,0,0.021092799305915833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,4,128,1,float16,float16,0,0.02075680047273636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,4,128,1,float16,fp8,0,0.021614399552345277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,4,128,1,fp8,fp8,0,0.02142080068588257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,8,128,1,float16,float16,0,0.02048960030078888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,8,128,1,float16,fp8,0,0.021459199488162994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,8,128,1,fp8,fp8,0,0.02128320038318634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,2,128,1,float16,float16,0,0.020411199331283568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,40,128,1,float16,float16,0,0.020971199870109557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,40,128,1,float16,fp8,0,0.021670399606227873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,4,128,1,float16,float16,0,0.02029760032892227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,40,128,1,fp8,fp8,0,0.02168480008840561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,1,128,1,float16,float16,0,0.020502400398254395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,1,128,1,float16,fp8,0,0.021320000290870667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,1,128,1,fp8,fp8,0,0.02131039947271347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,2,128,1,float16,fp8,0,0.021163199841976166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,2,128,1,fp8,fp8,0,0.02131360024213791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,4,128,1,float16,fp8,0,0.02120800018310547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,4,128,1,fp8,fp8,0,0.021035200357437132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,8,128,1,float16,float16,0,0.020417599380016326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,8,128,1,float16,fp8,0,0.021316799521446227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,8,128,1,fp8,fp8,0,0.021087999641895293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,1,128,1,float16,fp8,0,24.714199829101563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,1,128,1,fp8,fp8,0,24.791937255859374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,2,128,1,float16,fp8,0,24.97393798828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,2,128,1,fp8,fp8,0,24.97288360595703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,1,128,1,float16,float16,0,28.9592041015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,2,128,1,float16,float16,0,29.866656494140624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,4,128,1,float16,float16,0,30.226361083984376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,4,128,1,float16,fp8,0,25.59251251220703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,32,128,1,float16,fp8,0,16.057916259765626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,32,128,1,fp8,fp8,0,15.801594543457032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,32,128,1,float16,float16,0,18.79505615234375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,1,128,1,float16,float16,0,16.870140075683594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,4,128,1,fp8,fp8,0,27.244357299804687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,8,128,1,float16,fp8,0,27.829678344726563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,8,128,1,fp8,fp8,0,27.8430419921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,8,128,1,float16,float16,0,32.59809265136719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,1,128,1,float16,fp8,0,12.667110443115234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,1,128,1,fp8,fp8,0,12.468572998046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,2,128,1,float16,fp8,0,12.677635192871094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,2,128,1,float16,float16,0,14.6480224609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,2,128,1,fp8,fp8,0,12.532252502441406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,4,128,1,float16,fp8,0,12.652630615234376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,4,128,1,float16,float16,0,15.435169982910157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,4,128,1,fp8,fp8,0,12.612964630126953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,32,128,1,float16,float16,0,8.85862045288086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,32,128,1,float16,fp8,0,7.874700927734375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,8,128,1,float16,fp8,0,13.5175537109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,32,128,1,fp8,fp8,0,7.511100769042969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,8,128,1,float16,float16,0,16.284689331054686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,8,128,1,fp8,fp8,0,13.205064392089843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,1,128,1,float16,float16,0,7.707099151611328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,1,128,1,float16,fp8,0,6.671134185791016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,1,128,1,fp8,fp8,0,6.795732879638672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,2,128,1,float16,fp8,0,6.774750518798828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,2,128,1,float16,float16,0,8.162471771240234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,2,128,1,fp8,fp8,0,6.81710205078125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,4,128,1,float16,fp8,0,7.092040252685547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,4,128,1,fp8,fp8,0,6.826182556152344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,4,128,1,float16,float16,0,8.091932678222657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,32,128,1,float16,fp8,0,3.702803039550781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,32,128,1,float16,float16,0,4.348481750488281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,8,128,1,float16,fp8,0,6.595414733886718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,8,128,1,float16,float16,0,8.285539245605468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,32,128,1,fp8,fp8,0,3.912984085083008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,8,128,1,fp8,fp8,0,6.582355499267578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,1,128,1,float16,fp8,0,3.1681791305541993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,1,128,1,float16,float16,0,3.699224090576172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,1,128,1,fp8,fp8,0,3.175488090515137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,2,128,1,float16,fp8,0,3.13317756652832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,2,128,1,fp8,fp8,0,3.2360000610351562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,2,128,1,float16,float16,0,3.5994239807128907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,4,128,1,float16,fp8,0,3.1848384857177736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,4,128,1,fp8,fp8,0,3.304617691040039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,4,128,1,float16,float16,0,3.877916717529297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,8,128,1,float16,float16,0,3.9151615142822265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,8,128,1,float16,fp8,0,3.2632240295410155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,8,128,1,fp8,fp8,0,3.223075103759766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,1,128,1,float16,fp8,0,14.217483520507812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,1,128,1,fp8,fp8,0,14.322770690917968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,1,128,1,float16,float16,0,16.790525817871092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,2,128,1,float16,fp8,0,14.534329223632813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,2,128,1,fp8,fp8,0,14.331405639648438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,4,128,1,float16,fp8,0,14.576860046386718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,2,128,1,float16,float16,0,17.652577209472657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,4,128,1,float16,float16,0,17.379611206054687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,32,128,1,float16,fp8,0,9.116953277587891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,32,128,1,float16,float16,0,11.096260833740235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,32,128,1,fp8,fp8,0,8.947281646728516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,1,128,1,float16,float16,0,8.976509094238281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,4,128,1,fp8,fp8,0,15.695440673828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,8,128,1,float16,fp8,0,15.642739868164062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,8,128,1,fp8,fp8,0,16.106076049804688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,1,128,1,float16,fp8,0,7.2757408142089846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,8,128,1,float16,float16,0,18.87041015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,1,128,1,fp8,fp8,0,7.1875152587890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,2,128,1,float16,fp8,0,7.238251495361328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,2,128,1,fp8,fp8,0,7.427410888671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,2,128,1,float16,float16,0,9.011851501464843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,4,128,1,float16,fp8,0,7.898143768310547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,4,128,1,float16,float16,0,9.06542739868164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,4,128,1,fp8,fp8,0,7.260272216796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,32,128,1,float16,fp8,0,4.494361495971679
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,32,128,1,float16,float16,0,5.130372619628906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,8,128,1,float16,fp8,0,7.806483459472656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,8,128,1,float16,float16,0,9.041548919677734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,8,128,1,fp8,fp8,0,7.549793243408203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,32,128,1,fp8,fp8,0,4.381364822387695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,1,128,1,float16,float16,0,4.143601608276367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,1,128,1,float16,fp8,0,3.5352577209472655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,1,128,1,fp8,fp8,0,3.749505615234375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,2,128,1,float16,float16,0,4.058892822265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,2,128,1,float16,fp8,0,3.5815006256103517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,2,128,1,fp8,fp8,0,3.6411823272705077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,4,128,1,float16,float16,0,4.245256042480468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,4,128,1,float16,fp8,0,3.6774673461914062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,4,128,1,fp8,fp8,0,3.641836929321289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,8,128,1,float16,float16,0,4.487932968139648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,32,128,1,float16,float16,0,2.486382484436035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,8,128,1,float16,fp8,0,3.9568641662597654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,8,128,1,fp8,fp8,0,3.890884780883789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,32,128,1,float16,fp8,0,2.3796384811401365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,1,128,1,float16,float16,0,1.9456480026245118
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,32,128,1,fp8,fp8,0,2.205695915222168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,1,128,1,fp8,fp8,0,1.8216527938842773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,1,128,1,float16,fp8,0,1.9789840698242187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,2,128,1,float16,float16,0,2.4600688934326174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,2,128,1,float16,fp8,0,1.8081151962280273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,2,128,1,fp8,fp8,0,1.8410287857055665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,4,128,1,float16,fp8,0,1.8280879974365234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,4,128,1,float16,float16,0,2.158889579772949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,4,128,1,fp8,fp8,0,1.8305824279785157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,8,128,1,float16,float16,0,2.0624927520751952
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,8,128,1,fp8,fp8,0,1.8723600387573243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,8,128,1,float16,fp8,0,2.174675178527832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,1,128,1,float16,fp8,0,10.397200012207032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,1,128,1,fp8,fp8,0,10.061062622070313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,2,128,1,float16,fp8,0,10.61273422241211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,2,128,1,fp8,fp8,0,10.29989242553711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,1,128,1,float16,float16,0,12.336144256591798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,2,128,1,float16,float16,0,12.065187072753906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,4,128,1,float16,fp8,0,11.305579376220702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,4,128,1,float16,float16,0,12.966938781738282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,32,128,1,float16,fp8,0,6.366368103027344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,32,128,1,float16,float16,0,7.54583969116211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,32,128,1,fp8,fp8,0,6.589740753173828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,1,128,1,float16,float16,0,6.086415863037109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,4,128,1,fp8,fp8,0,11.048817443847657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,8,128,1,float16,fp8,0,10.760233306884766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,8,128,1,fp8,fp8,0,10.692479705810547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,8,128,1,float16,float16,0,13.200337219238282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,1,128,1,float16,fp8,0,4.980926513671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,1,128,1,fp8,fp8,0,5.000260925292968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,2,128,1,float16,fp8,0,5.225727844238281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,2,128,1,float16,float16,0,6.70110855102539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,2,128,1,fp8,fp8,0,5.295884704589843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,4,128,1,float16,fp8,0,5.113395309448242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,4,128,1,float16,float16,0,6.7316337585449215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,4,128,1,fp8,fp8,0,5.2054798126220705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,8,128,1,float16,float16,0,6.257118225097656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,8,128,1,float16,fp8,0,5.296529769897461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,32,128,1,float16,float16,0,3.863974380493164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,32,128,1,float16,fp8,0,3.2081695556640626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,32,128,1,fp8,fp8,0,3.1882896423339844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,8,128,1,fp8,fp8,0,5.353286361694336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,1,128,1,float16,float16,0,2.971451187133789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,1,128,1,float16,fp8,0,2.50447998046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,1,128,1,fp8,fp8,0,2.6669328689575194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,2,128,1,float16,fp8,0,2.5781679153442383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,2,128,1,fp8,fp8,0,2.5260671615600585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,2,128,1,float16,float16,0,2.9319759368896485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,4,128,1,float16,fp8,0,2.579158401489258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,4,128,1,float16,float16,0,3.0030832290649414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,4,128,1,fp8,fp8,0,2.6302911758422853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,8,128,1,float16,float16,0,3.1458303451538088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,32,128,1,float16,float16,0,1.8218704223632813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,32,128,1,float16,fp8,0,1.6649599075317383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,32,128,1,fp8,fp8,0,1.6281312942504882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,1,128,1,float16,float16,0,1.5928863525390624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,1,128,1,float16,fp8,0,1.2665632247924805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,8,128,1,float16,fp8,0,3.017393684387207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,8,128,1,fp8,fp8,0,2.783316802978516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,1,128,1,fp8,fp8,0,1.5150239944458008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,2,128,1,float16,float16,0,1.429032039642334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,2,128,1,float16,fp8,0,1.2953328132629394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,2,128,1,fp8,fp8,0,1.2669584274291992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,4,128,1,float16,fp8,0,1.2991423606872559
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,4,128,1,float16,float16,0,1.689321517944336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,4,128,1,fp8,fp8,0,1.334001636505127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,8,128,1,float16,fp8,0,1.379366397857666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,8,128,1,float16,float16,0,1.606118392944336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,8,128,1,fp8,fp8,0,1.4935248374938965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,1,128,1,float16,fp8,0,13.454081726074218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,1,128,1,fp8,fp8,0,13.194386291503907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,2,128,1,fp8,fp8,0,13.123684692382813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,2,128,1,float16,fp8,0,13.480584716796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,1,128,1,float16,float16,0,15.597331237792968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,4,128,1,float16,fp8,0,13.746417236328124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,2,128,1,float16,float16,0,15.82010498046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,4,128,1,float16,float16,0,16.941261291503906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,32,128,1,float16,fp8,0,8.907150268554688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,32,128,1,fp8,fp8,0,8.632653045654298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,32,128,1,float16,float16,0,10.061793518066406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,1,128,1,float16,float16,0,8.1289794921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,4,128,1,fp8,fp8,0,13.974504089355468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,8,128,1,fp8,fp8,0,13.977017211914063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,8,128,1,float16,fp8,0,14.535789489746094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,8,128,1,float16,float16,0,17.327424621582033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,1,128,1,float16,fp8,0,6.938587188720703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,1,128,1,fp8,fp8,0,6.4901885986328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,2,128,1,float16,fp8,0,7.237651062011719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,2,128,1,float16,float16,0,8.405715179443359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,2,128,1,fp8,fp8,0,6.7965950012207035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,4,128,1,float16,fp8,0,6.864044952392578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,4,128,1,float16,float16,0,8.754502105712891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,4,128,1,fp8,fp8,0,6.87524642944336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,32,128,1,float16,float16,0,5.101507186889648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,8,128,1,float16,fp8,0,7.181974029541015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,32,128,1,float16,fp8,0,4.4223777770996096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,8,128,1,float16,float16,0,9.263174438476563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,8,128,1,fp8,fp8,0,7.912179565429687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,32,128,1,fp8,fp8,0,4.501625442504883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,1,128,1,float16,float16,0,3.819414520263672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,1,128,1,float16,fp8,0,3.3273681640625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,1,128,1,fp8,fp8,0,3.2574798583984377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,2,128,1,float16,fp8,0,3.350038528442383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,2,128,1,float16,float16,0,3.9584735870361327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,2,128,1,fp8,fp8,0,3.565848159790039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,4,128,1,float16,fp8,0,3.3721614837646485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,4,128,1,float16,float16,0,3.783406448364258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,4,128,1,fp8,fp8,0,3.4011089324951174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,8,128,1,float16,fp8,0,3.495550537109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,8,128,1,float16,float16,0,4.139678573608398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,32,128,1,float16,float16,0,2.4842016220092775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,32,128,1,float16,fp8,0,2.208603286743164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,8,128,1,fp8,fp8,0,3.512955093383789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,32,128,1,fp8,fp8,0,2.189132881164551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,1,128,1,float16,fp8,0,1.6422079086303711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,1,128,1,float16,float16,0,1.9097856521606444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,1,128,1,fp8,fp8,0,1.6332000732421874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,2,128,1,float16,float16,0,1.9151424407958983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,2,128,1,float16,fp8,0,1.6973583221435546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,2,128,1,fp8,fp8,0,1.7854991912841798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,4,128,1,float16,fp8,0,1.6872608184814453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,4,128,1,fp8,fp8,0,1.6982767105102539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,4,128,1,float16,float16,0,1.9264608383178712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,8,128,1,float16,float16,0,2.0573312759399416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,8,128,1,float16,fp8,0,1.7465072631835938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,32,128,1,float16,float16,0,1.1974736213684083
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,32,128,1,float16,fp8,0,1.1484127998352052
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,1,128,1,float16,float16,0,0.9052240371704101
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,32,128,1,fp8,fp8,0,1.1266575813293458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,1,128,1,float16,fp8,0,0.9120575904846191
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,8,128,1,fp8,fp8,0,2.031532859802246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,1,128,1,fp8,fp8,0,0.8913760185241699
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,2,128,1,float16,float16,0,0.8923151969909668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,2,128,1,float16,fp8,0,0.8752448081970214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,2,128,1,fp8,fp8,0,0.8606736183166503
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,4,128,1,float16,float16,0,0.9358127593994141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,4,128,1,float16,fp8,0,0.8821056365966797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,4,128,1,fp8,fp8,0,0.8965968132019043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,8,128,1,float16,float16,0,0.9658975601196289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,8,128,1,float16,fp8,0,0.8978735923767089
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,8,128,1,fp8,fp8,0,0.9574928283691406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,1,128,1,float16,fp8,0,7.846943664550781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,1,128,1,fp8,fp8,0,7.5939170837402346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,2,128,1,float16,fp8,0,7.78310546875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,2,128,1,fp8,fp8,0,7.6939857482910154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,1,128,1,float16,float16,0,8.839401245117188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,4,128,1,float16,fp8,0,7.870731353759766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,2,128,1,float16,float16,0,9.061151885986328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,4,128,1,float16,float16,0,9.282035064697265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,1,128,1,float16,float16,0,4.186816024780273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,32,128,1,float16,fp8,0,5.390996932983398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,32,128,1,float16,float16,0,6.128628921508789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,32,128,1,fp8,fp8,0,5.405472183227539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,4,128,1,fp8,fp8,0,8.258321380615234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,8,128,1,float16,fp8,0,8.299664306640626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,8,128,1,fp8,fp8,0,8.253431701660157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,8,128,1,float16,float16,0,9.915070343017579
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,1,128,1,float16,fp8,0,3.9139694213867187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,1,128,1,fp8,fp8,0,3.763382339477539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,2,128,1,float16,fp8,0,3.829568099975586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,2,128,1,float16,float16,0,4.546988677978516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,2,128,1,fp8,fp8,0,4.050352096557617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,4,128,1,float16,fp8,0,3.9346897125244142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,4,128,1,float16,float16,0,4.771950531005859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,4,128,1,fp8,fp8,0,4.054420852661133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,8,128,1,float16,fp8,0,4.118780899047851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,32,128,1,float16,float16,0,3.0688688278198244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,32,128,1,float16,fp8,0,2.7478736877441405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,8,128,1,fp8,fp8,0,4.1957744598388675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,8,128,1,float16,float16,0,5.387707138061524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,32,128,1,fp8,fp8,0,2.7299503326416015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,1,128,1,float16,fp8,0,1.9032672882080077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,1,128,1,float16,float16,0,2.4377840042114256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,1,128,1,fp8,fp8,0,1.8796096801757813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,2,128,1,float16,fp8,0,1.959124755859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,2,128,1,float16,float16,0,2.204350471496582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,2,128,1,fp8,fp8,0,1.9323551177978515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,4,128,1,float16,fp8,0,1.9725919723510743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,4,128,1,float16,float16,0,2.288400077819824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,4,128,1,fp8,fp8,0,1.9820703506469726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,8,128,1,float16,float16,0,2.4149871826171876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,32,128,1,float16,fp8,0,1.3793968200683593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,32,128,1,float16,float16,0,1.5431695938110352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,8,128,1,float16,fp8,0,2.060647964477539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,1,128,1,float16,float16,0,1.1519776344299317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,32,128,1,fp8,fp8,0,1.3772656440734863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,1,128,1,float16,fp8,0,1.0172063827514648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,8,128,1,fp8,fp8,0,2.1260671615600586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,1,128,1,fp8,fp8,0,1.0438287734985352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,2,128,1,float16,fp8,0,1.067403221130371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,2,128,1,float16,float16,0,1.039515209197998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,2,128,1,fp8,fp8,0,0.9686176300048828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,4,128,1,float16,float16,0,1.1062944412231446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,4,128,1,float16,fp8,0,1.0182784080505372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,4,128,1,fp8,fp8,0,1.0267168045043946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,8,128,1,float16,float16,0,1.1507712364196778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,8,128,1,float16,fp8,0,1.0554896354675294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,32,128,1,float16,float16,0,0.7589471817016602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,8,128,1,fp8,fp8,0,1.0533807754516602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,32,128,1,float16,fp8,0,0.7314320087432862
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,1,128,1,float16,float16,0,0.5239840030670166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,1,128,1,float16,fp8,0,0.5171360015869141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,32,128,1,fp8,fp8,0,0.7181263923645019
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,1,128,1,fp8,fp8,0,0.4968656063079834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,2,128,1,float16,float16,0,0.52259202003479
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,2,128,1,float16,fp8,0,0.5009552001953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,2,128,1,fp8,fp8,0,0.5032959938049316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,4,128,1,float16,float16,0,0.547156810760498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,8,128,1,float16,float16,0,0.5793087959289551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,4,128,1,float16,fp8,0,0.5191376209259033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,4,128,1,fp8,fp8,0,0.5252607822418213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,8,128,1,float16,fp8,0,0.5357808113098145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,8,128,1,fp8,fp8,0,0.5377439975738525
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,1,128,1,float16,fp8,0,7.173358154296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,1,128,1,fp8,fp8,0,7.2442787170410154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,1,128,1,float16,float16,0,8.258819580078125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,2,128,1,float16,fp8,0,7.250857543945313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,2,128,1,fp8,fp8,0,7.337067413330078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,2,128,1,float16,float16,0,8.514628601074218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,4,128,1,float16,float16,0,8.914260864257812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,4,128,1,float16,fp8,0,7.557849884033203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,32,128,1,float16,float16,0,6.258878326416015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,32,128,1,float16,fp8,0,5.790742492675781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,4,128,1,fp8,fp8,0,7.5276031494140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,1,128,1,float16,float16,0,4.087534332275391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,32,128,1,fp8,fp8,0,5.764616012573242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,8,128,1,float16,fp8,0,8.065325164794922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,8,128,1,fp8,fp8,0,8.05709457397461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,8,128,1,float16,float16,0,9.32119369506836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,1,128,1,float16,fp8,0,3.6866992950439452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,1,128,1,fp8,fp8,0,3.7302623748779298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,2,128,1,float16,float16,0,4.157918548583984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,2,128,1,float16,fp8,0,3.6408031463623045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,2,128,1,fp8,fp8,0,3.6344577789306642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,4,128,1,float16,fp8,0,3.768560028076172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,4,128,1,float16,float16,0,4.3042560577392575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,4,128,1,fp8,fp8,0,3.861017608642578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,32,128,1,float16,float16,0,3.148574447631836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,1,128,1,float16,fp8,0,1.7914863586425782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,32,128,1,float16,fp8,0,2.868976020812988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,1,128,1,float16,float16,0,1.9995119094848632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,8,128,1,float16,float16,0,4.557601547241211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,8,128,1,float16,fp8,0,4.271811294555664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,32,128,1,fp8,fp8,0,2.8822303771972657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,8,128,1,fp8,fp8,0,4.121737670898438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,1,128,1,fp8,fp8,0,1.8452943801879882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,2,128,1,float16,float16,0,2.034814453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,2,128,1,float16,fp8,0,1.8112415313720702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,2,128,1,fp8,fp8,0,1.9268672943115235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,4,128,1,float16,fp8,0,1.8929071426391602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,4,128,1,float16,float16,0,2.062843132019043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,4,128,1,fp8,fp8,0,1.8926528930664062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,8,128,1,float16,float16,0,2.251380729675293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,8,128,1,float16,fp8,0,2.0725536346435547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,32,128,1,float16,float16,0,1.5694543838500976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,32,128,1,float16,fp8,0,1.4644271850585937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,1,128,1,float16,float16,0,0.9743231773376465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,8,128,1,fp8,fp8,0,2.128099250793457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,1,128,1,float16,fp8,0,0.909160041809082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,32,128,1,fp8,fp8,0,1.464891242980957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,1,128,1,fp8,fp8,0,0.9747648239135742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,2,128,1,float16,float16,0,0.9806127548217773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,2,128,1,float16,fp8,0,0.9582304000854492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,2,128,1,fp8,fp8,0,0.9318143844604492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,4,128,1,float16,float16,0,1.0364303588867188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,4,128,1,float16,fp8,0,0.9825296401977539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,4,128,1,fp8,fp8,0,0.960694408416748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,8,128,1,float16,float16,0,1.1017439842224122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,8,128,1,float16,fp8,0,1.0679072380065917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,32,128,1,float16,float16,0,0.8002207756042481
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,8,128,1,fp8,fp8,0,1.070684814453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,32,128,1,float16,fp8,0,0.743828821182251
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,1,128,1,float16,float16,0,0.5044591903686524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,32,128,1,fp8,fp8,0,0.7773392200469971
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,1,128,1,float16,fp8,0,0.46990399360656737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,1,128,1,fp8,fp8,0,0.4709472179412842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,2,128,1,float16,float16,0,0.5074880123138428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,2,128,1,float16,fp8,0,0.4956048011779785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,2,128,1,fp8,fp8,0,0.48128318786621094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,4,128,1,float16,float16,0,0.519217586517334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,4,128,1,float16,fp8,0,0.5074160099029541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,4,128,1,fp8,fp8,0,0.5162047863006591
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,8,128,1,float16,float16,0,0.5627823829650879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,8,128,1,float16,fp8,0,0.5268991947174072
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,32,128,1,float16,float16,0,0.4130959987640381
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,8,128,1,fp8,fp8,0,0.5412752151489257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,32,128,1,float16,fp8,0,0.39315199851989746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,2,128,1,float16,float16,0,0.26167519092559816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,32,128,1,fp8,fp8,0,0.39236481189727784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,1,128,1,float16,float16,0,0.26440000534057617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,1,128,1,float16,fp8,0,0.25248639583587645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,1,128,1,fp8,fp8,0,0.2522480010986328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,2,128,1,float16,fp8,0,0.25805439949035647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,2,128,1,fp8,fp8,0,0.26063520908355714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,4,128,1,float16,float16,0,0.275711989402771
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,4,128,1,float16,fp8,0,0.2606656074523926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,4,128,1,fp8,fp8,0,0.2664639949798584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,8,128,1,float16,float16,0,0.2959136009216309
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,8,128,1,float16,fp8,0,0.28389439582824705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,8,128,1,fp8,fp8,0,0.2839440107345581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,1,128,1,float16,fp8,0,4.320553588867187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,1,128,1,fp8,fp8,0,4.296393585205078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,1,128,1,float16,float16,0,4.857758331298828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,2,128,1,float16,fp8,0,4.465588760375977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,2,128,1,float16,float16,0,4.751217651367187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,2,128,1,fp8,fp8,0,4.3822479248046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,4,128,1,float16,float16,0,5.099937438964844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,4,128,1,float16,fp8,0,4.558740615844727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,4,128,1,fp8,fp8,0,4.564108657836914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,32,128,1,float16,float16,0,4.083718490600586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,32,128,1,float16,fp8,0,3.93451042175293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,8,128,1,float16,fp8,0,4.945407867431641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,32,128,1,fp8,fp8,0,3.828472137451172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,8,128,1,fp8,fp8,0,4.975894546508789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,8,128,1,float16,float16,0,5.618206405639649
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,1,128,1,float16,float16,0,2.3544464111328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,1,128,1,float16,fp8,0,2.2356496810913087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,1,128,1,fp8,fp8,0,2.1625152587890626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,2,128,1,float16,fp8,0,2.1638256072998048
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,2,128,1,float16,float16,0,2.4044000625610353
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,2,128,1,fp8,fp8,0,2.1852527618408204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,4,128,1,float16,float16,0,2.548219108581543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,4,128,1,float16,fp8,0,2.4585855484008787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,4,128,1,fp8,fp8,0,2.442736053466797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,8,128,1,float16,fp8,0,2.520035171508789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,1,128,1,float16,float16,0,1.1399151802062988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,8,128,1,float16,float16,0,2.966241645812988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,32,128,1,float16,float16,0,2.123521614074707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,32,128,1,float16,fp8,0,1.9195728302001953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,8,128,1,fp8,fp8,0,2.485032081604004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,32,128,1,fp8,fp8,0,2.0097631454467773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,1,128,1,float16,fp8,0,1.0801759719848634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,1,128,1,fp8,fp8,0,1.0801136016845703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,2,128,1,float16,float16,0,1.167849636077881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,2,128,1,float16,fp8,0,1.2301376342773438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,2,128,1,fp8,fp8,0,1.1599679946899415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,4,128,1,float16,float16,0,1.2391776084899901
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,4,128,1,float16,fp8,0,1.146401596069336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,4,128,1,fp8,fp8,0,1.1788496017456054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,8,128,1,float16,float16,0,1.3314895629882812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,8,128,1,float16,fp8,0,1.266260814666748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,8,128,1,fp8,fp8,0,1.3156399726867676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,32,128,1,float16,float16,0,1.0531984329223634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,32,128,1,float16,fp8,0,0.987713623046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,1,128,1,float16,float16,0,0.5695312023162842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,32,128,1,fp8,fp8,0,0.97293119430542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,1,128,1,float16,fp8,0,0.555017614364624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,1,128,1,fp8,fp8,0,0.5713871955871582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,2,128,1,float16,float16,0,0.5954944133758545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,4,128,1,float16,fp8,0,0.5879712104797363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,2,128,1,float16,fp8,0,0.5812607765197754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,2,128,1,fp8,fp8,0,0.5684175968170166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,4,128,1,float16,float16,0,0.6246880054473877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,4,128,1,fp8,fp8,0,0.6049536228179931
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,8,128,1,float16,float16,0,0.6752031803131103
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,8,128,1,float16,fp8,0,0.6567935943603516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,8,128,1,fp8,fp8,0,0.6543280124664307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,32,128,1,float16,float16,0,0.5234032154083252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,1,128,1,fp8,fp8,0,0.28842079639434814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,32,128,1,float16,fp8,0,0.499510383605957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,32,128,1,fp8,fp8,0,0.5079775810241699
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,1,128,1,float16,float16,0,0.3018784046173096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,1,128,1,float16,fp8,0,0.29409279823303225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,2,128,1,float16,float16,0,0.31101601123809813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,2,128,1,float16,fp8,0,0.2990416049957275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,2,128,1,fp8,fp8,0,0.29372639656066896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,4,128,1,float16,float16,0,0.32536959648132324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,4,128,1,float16,fp8,0,0.31106719970703123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,4,128,1,fp8,fp8,0,0.3159519910812378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,8,128,1,float16,float16,0,0.3540031909942627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,8,128,1,float16,fp8,0,0.341975998878479
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,8,128,1,fp8,fp8,0,0.3424079895019531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,32,128,1,float16,float16,0,0.2846127986907959
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,32,128,1,float16,fp8,0,0.2651695966720581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,2,128,1,float16,fp8,0,0.16342560052871705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,32,128,1,fp8,fp8,0,0.26442561149597166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,1,128,1,float16,float16,0,0.16347999572753907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,1,128,1,float16,fp8,0,0.1615823984146118
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,1,128,1,fp8,fp8,0,0.1616528034210205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,2,128,1,float16,float16,0,0.16860640048980713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,2,128,1,fp8,fp8,0,0.1626736044883728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,4,128,1,float16,float16,0,0.17512320280075072
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,4,128,1,float16,fp8,0,0.1696895956993103
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,4,128,1,fp8,fp8,0,0.16948800086975097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,8,128,1,float16,float16,0,0.1859663963317871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,8,128,1,float16,fp8,0,0.1843183994293213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,8,128,1,fp8,fp8,0,0.18257280588150024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,1,128,1,float16,fp8,0,4.232767868041992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,1,128,1,fp8,fp8,0,4.225688171386719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,1,128,1,float16,float16,0,4.751488113403321
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,2,128,1,float16,fp8,0,4.3559009552001955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,2,128,1,fp8,fp8,0,4.406537628173828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,2,128,1,float16,float16,0,4.775096130371094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,4,128,1,float16,fp8,0,4.631798553466797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,4,128,1,float16,float16,0,5.191923141479492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,4,128,1,fp8,fp8,0,4.600067138671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,32,128,1,float16,float16,0,4.554608154296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,8,128,1,float16,fp8,0,5.1378944396972654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,32,128,1,float16,fp8,0,4.294699096679688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,8,128,1,float16,float16,0,5.713297653198242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,8,128,1,fp8,fp8,0,5.155315017700195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,32,128,1,fp8,fp8,0,4.391110229492187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,1,128,1,float16,float16,0,2.267862319946289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,1,128,1,float16,fp8,0,2.1158367156982423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,1,128,1,fp8,fp8,0,2.1506912231445314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,2,128,1,float16,float16,0,2.316454315185547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,2,128,1,float16,fp8,0,2.1636127471923827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,2,128,1,fp8,fp8,0,2.185628890991211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,4,128,1,float16,fp8,0,2.3238384246826174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,4,128,1,float16,float16,0,2.524718475341797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,4,128,1,fp8,fp8,0,2.428883171081543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,1,128,1,float16,float16,0,1.1789792060852051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,8,128,1,float16,fp8,0,2.599033546447754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,8,128,1,float16,float16,0,2.8633424758911135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,1,128,1,float16,fp8,0,1.087451171875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,32,128,1,float16,float16,0,2.27052001953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,32,128,1,float16,fp8,0,2.17435359954834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,32,128,1,fp8,fp8,0,2.177124786376953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,8,128,1,fp8,fp8,0,2.7108863830566405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,1,128,1,fp8,fp8,0,1.131766414642334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,2,128,1,float16,float16,0,1.1500880241394043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,2,128,1,float16,fp8,0,1.1467743873596192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,2,128,1,fp8,fp8,0,1.1031791687011718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,4,128,1,float16,float16,0,1.2405280113220214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,4,128,1,float16,fp8,0,1.1716400146484376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,4,128,1,fp8,fp8,0,1.1711456298828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,8,128,1,float16,float16,0,1.3790656089782716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,8,128,1,float16,fp8,0,1.347268772125244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,1,128,1,float16,float16,0,0.5710207939147949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,8,128,1,fp8,fp8,0,1.3408927917480469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,32,128,1,float16,float16,0,1.1791215896606446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,32,128,1,fp8,fp8,0,1.1352895736694335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,32,128,1,float16,fp8,0,1.1053088188171387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,1,128,1,float16,fp8,0,0.5461423873901368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,1,128,1,fp8,fp8,0,0.5461056232452393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,2,128,1,float16,float16,0,0.5852928161621094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,2,128,1,float16,fp8,0,0.5770832061767578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,2,128,1,fp8,fp8,0,0.5761392116546631
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,4,128,1,float16,float16,0,0.638102388381958
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,4,128,1,float16,fp8,0,0.6100656032562256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,8,128,1,float16,fp8,0,0.6677279949188233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,4,128,1,fp8,fp8,0,0.5992991924285889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,8,128,1,float16,float16,0,0.7071167945861816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,8,128,1,fp8,fp8,0,0.6678544044494629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,32,128,1,float16,float16,0,0.6007455825805664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,32,128,1,float16,fp8,0,0.5765456199645996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,2,128,1,float16,float16,0,0.30715360641479494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,1,128,1,float16,float16,0,0.3001823902130127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,32,128,1,fp8,fp8,0,0.5779952049255371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,1,128,1,float16,fp8,0,0.29286239147186277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,1,128,1,fp8,fp8,0,0.2938607931137085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,2,128,1,float16,fp8,0,0.2951535940170288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,2,128,1,fp8,fp8,0,0.30171520709991456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,4,128,1,float16,float16,0,0.32888801097869874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,4,128,1,float16,fp8,0,0.31755039691925047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,4,128,1,fp8,fp8,0,0.31922240257263185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,8,128,1,float16,float16,0,0.37073919773101804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,8,128,1,float16,fp8,0,0.3531071901321411
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,1,128,1,float16,fp8,0,0.1588912010192871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,1,128,1,fp8,fp8,0,0.15879039764404296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,8,128,1,fp8,fp8,0,0.35316479206085205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,32,128,1,float16,float16,0,0.3102031946182251
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,32,128,1,float16,fp8,0,0.3005183935165405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,32,128,1,fp8,fp8,0,0.30010080337524414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,1,128,1,float16,float16,0,0.16054079532623292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,4,128,1,fp8,fp8,0,0.17248640060424805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,2,128,1,float16,float16,0,0.16769759654998778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,2,128,1,float16,fp8,0,0.1642367959022522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,2,128,1,fp8,fp8,0,0.16635680198669434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,4,128,1,float16,float16,0,0.17754559516906737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,4,128,1,float16,fp8,0,0.16851520538330078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,8,128,1,float16,float16,0,0.19699519872665405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,8,128,1,float16,fp8,0,0.18954240083694457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,8,128,1,fp8,fp8,0,0.18872159719467163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,32,128,1,float16,float16,0,0.16980799436569213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,2,128,1,float16,float16,0,0.09130719900131226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,32,128,1,float16,fp8,0,0.1619488000869751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,32,128,1,fp8,fp8,0,0.16174559593200682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,1,128,1,float16,float16,0,0.08922079801559449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,1,128,1,float16,fp8,0,0.08441119790077209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,1,128,1,fp8,fp8,0,0.08489279747009278
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,2,128,1,float16,fp8,0,0.08452159762382508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,2,128,1,fp8,fp8,0,0.08528000116348267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,4,128,1,float16,float16,0,0.09970880150794983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,4,128,1,float16,fp8,0,0.09144160151481628
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,4,128,1,fp8,fp8,0,0.09059839844703674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,8,128,1,float16,float16,0,0.10957920551300049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,8,128,1,float16,fp8,0,0.10174399614334106
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,8,128,1,fp8,fp8,0,0.10157920122146606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,1,128,1,float16,float16,0,2.6837392807006837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,1,128,1,float16,fp8,0,2.6226640701293946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,1,128,1,fp8,fp8,0,2.6191328048706053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,2,128,1,float16,float16,0,2.8837135314941404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,2,128,1,float16,fp8,0,2.740718460083008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,2,128,1,fp8,fp8,0,2.7322271347045897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,4,128,1,float16,fp8,0,2.97652645111084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,4,128,1,float16,float16,0,3.106831932067871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,4,128,1,fp8,fp8,0,2.934556770324707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,8,128,1,float16,float16,0,3.586355209350586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,32,128,1,float16,float16,0,3.1370351791381834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,8,128,1,float16,fp8,0,3.3679119110107423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,8,128,1,fp8,fp8,0,3.3442081451416015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,1,128,1,float16,float16,0,1.3490639686584474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,32,128,1,float16,fp8,0,3.016763114929199
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,32,128,1,fp8,fp8,0,3.0021568298339845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,1,128,1,float16,fp8,0,1.3569071769714356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,1,128,1,fp8,fp8,0,1.3296143531799316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,2,128,1,float16,float16,0,1.403758430480957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,2,128,1,float16,fp8,0,1.366806411743164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,2,128,1,fp8,fp8,0,1.3784768104553222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,4,128,1,float16,float16,0,1.5318191528320313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,4,128,1,float16,fp8,0,1.4783807754516602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,4,128,1,fp8,fp8,0,1.4778672218322755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,8,128,1,float16,float16,0,1.7653680801391602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,8,128,1,float16,fp8,0,1.7415935516357421
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,8,128,1,fp8,fp8,0,1.6771984100341797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,32,128,1,float16,float16,0,1.5624112129211425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,1,128,1,float16,float16,0,0.6866943836212158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,32,128,1,float16,fp8,0,1.5576383590698242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,32,128,1,fp8,fp8,0,1.5134032249450684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,1,128,1,float16,fp8,0,0.6775856018066406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,1,128,1,fp8,fp8,0,0.684827184677124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,2,128,1,float16,float16,0,0.717844820022583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,2,128,1,float16,fp8,0,0.7108816146850586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,2,128,1,fp8,fp8,0,0.7096335887908936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,4,128,1,float16,float16,0,0.7864528179168702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,4,128,1,float16,fp8,0,0.7503920078277588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,4,128,1,fp8,fp8,0,0.7589407920837402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,8,128,1,float16,float16,0,0.8940688133239746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,8,128,1,float16,fp8,0,0.8714256286621094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,32,128,1,float16,float16,0,0.8084976196289062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,8,128,1,fp8,fp8,0,0.8503664016723633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,1,128,1,float16,float16,0,0.35311839580535886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,32,128,1,float16,fp8,0,0.7889423847198487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,1,128,1,float16,fp8,0,0.35180160999298093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,32,128,1,fp8,fp8,0,0.7859055995941162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,1,128,1,fp8,fp8,0,0.35146241188049315
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,2,128,1,float16,float16,0,0.3765376091003418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,2,128,1,float16,fp8,0,0.3680351972579956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,2,128,1,fp8,fp8,0,0.3622416019439697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,4,128,1,float16,float16,0,0.3964128017425537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,4,128,1,float16,fp8,0,0.3947727918624878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,4,128,1,fp8,fp8,0,0.3922703981399536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,8,128,1,float16,float16,0,0.46173920631408694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,8,128,1,float16,fp8,0,0.44290881156921386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,8,128,1,fp8,fp8,0,0.44803519248962403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,32,128,1,float16,float16,0,0.42012639045715333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,32,128,1,float16,fp8,0,0.40297441482543944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,32,128,1,fp8,fp8,0,0.3983536005020142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,1,128,1,float16,float16,0,0.19169280529022217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,1,128,1,float16,fp8,0,0.18546080589294434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,1,128,1,fp8,fp8,0,0.1891055941581726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,2,128,1,float16,float16,0,0.19865280389785767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,2,128,1,float16,fp8,0,0.1975823998451233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,2,128,1,fp8,fp8,0,0.19629280567169188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,4,128,1,float16,float16,0,0.21339681148529052
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,4,128,1,float16,fp8,0,0.20578079223632811
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,4,128,1,fp8,fp8,0,0.2101439952850342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,8,128,1,float16,float16,0,0.2399888038635254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,8,128,1,float16,fp8,0,0.23562240600585938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,8,128,1,fp8,fp8,0,0.23438720703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,32,128,1,float16,float16,0,0.2225584030151367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,32,128,1,float16,fp8,0,0.21666719913482665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,2,128,1,float16,fp8,0,0.10819679498672485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,32,128,1,fp8,fp8,0,0.21590878963470458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,1,128,1,float16,float16,0,0.1061519980430603
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,1,128,1,float16,fp8,0,0.10684319734573364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,1,128,1,fp8,fp8,0,0.1049888014793396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,2,128,1,float16,float16,0,0.10984959602355956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,8,128,1,float16,fp8,0,0.13038400411605836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,2,128,1,fp8,fp8,0,0.10867040157318116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,4,128,1,float16,float16,0,0.1183616042137146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,4,128,1,float16,fp8,0,0.11612639427185059
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,4,128,1,fp8,fp8,0,0.11561759710311889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,8,128,1,float16,float16,0,0.13284800052642823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,8,128,1,fp8,fp8,0,0.12827999591827394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,1,128,1,fp8,fp8,0,0.0597055971622467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,32,128,1,float16,float16,0,0.12267839908599854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,32,128,1,float16,fp8,0,0.11920640468597413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,32,128,1,fp8,fp8,0,0.11743359565734864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,1,128,1,float16,float16,0,0.060889601707458496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,1,128,1,float16,fp8,0,0.05914880037307739
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,2,128,1,float16,float16,0,0.061857599020004275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,2,128,1,float16,fp8,0,0.05931040048599243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,2,128,1,fp8,fp8,0,0.05940799713134766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,4,128,1,float16,float16,0,0.064547199010849
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,4,128,1,float16,fp8,0,0.06253759860992432
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,4,128,1,fp8,fp8,0,0.06232320070266724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,8,128,1,float16,float16,0,0.07551040053367615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,8,128,1,float16,fp8,0,0.06860960125923157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,8,128,1,fp8,fp8,0,0.06966720223426819
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,1,128,1,float16,float16,0,2.8525615692138673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,1,128,1,float16,fp8,0,2.7645488739013673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,1,128,1,fp8,fp8,0,2.7725887298583984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,2,128,1,float16,float16,0,3.0414592742919924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,2,128,1,float16,fp8,0,2.910432052612305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,2,128,1,fp8,fp8,0,2.9160367965698244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,4,128,1,float16,float16,0,3.350014495849609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,4,128,1,float16,fp8,0,3.173137664794922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,4,128,1,fp8,fp8,0,3.181155204772949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,8,128,1,float16,float16,0,3.8821792602539062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,8,128,1,float16,fp8,0,3.7417407989501954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,8,128,1,fp8,fp8,0,3.7373023986816407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,32,128,1,float16,float16,0,3.8073360443115236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,1,128,1,float16,float16,0,1.4168304443359374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,32,128,1,float16,fp8,0,3.6384639739990234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,1,128,1,float16,fp8,0,1.3920751571655274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,1,128,1,fp8,fp8,0,1.3851119995117187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,2,128,1,float16,float16,0,1.4846351623535157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,2,128,1,float16,fp8,0,1.4754719734191895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,2,128,1,fp8,fp8,0,1.5026335716247559
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,32,128,1,fp8,fp8,0,3.583638381958008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,4,128,1,float16,float16,0,1.6680831909179688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,4,128,1,float16,fp8,0,1.6133951187133788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,4,128,1,fp8,fp8,0,1.634833526611328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,1,128,1,float16,float16,0,0.7272496223449707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,8,128,1,float16,float16,0,1.963199996948242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,8,128,1,float16,fp8,0,1.8815679550170898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,8,128,1,fp8,fp8,0,1.8858959197998046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,1,128,1,float16,fp8,0,0.7216224193572998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,32,128,1,float16,float16,0,1.9289743423461914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,32,128,1,float16,fp8,0,1.8214672088623047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,32,128,1,fp8,fp8,0,1.840665626525879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,1,128,1,fp8,fp8,0,0.710478401184082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,2,128,1,float16,float16,0,0.7875984191894532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,2,128,1,float16,fp8,0,0.7586976051330566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,2,128,1,fp8,fp8,0,0.7433072090148926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,4,128,1,float16,float16,0,0.8395215988159179
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,4,128,1,float16,fp8,0,0.8253600120544433
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,4,128,1,fp8,fp8,0,0.812161636352539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,8,128,1,float16,float16,0,0.9868927955627441
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,8,128,1,float16,fp8,0,0.9515487670898437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,8,128,1,fp8,fp8,0,0.9735856056213379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,32,128,1,float16,float16,0,0.9791119575500489
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,1,128,1,float16,float16,0,0.37942240238189695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,1,128,1,float16,fp8,0,0.36580960750579833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,32,128,1,float16,fp8,0,0.9232175827026368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,32,128,1,fp8,fp8,0,0.9283535957336426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,1,128,1,fp8,fp8,0,0.3657824039459229
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,2,128,1,float16,float16,0,0.3886800050735474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,2,128,1,float16,fp8,0,0.39019839763641356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,2,128,1,fp8,fp8,0,0.39208159446716306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,4,128,1,float16,float16,0,0.4370607852935791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,4,128,1,float16,fp8,0,0.4174352169036865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,4,128,1,fp8,fp8,0,0.4169007778167725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,8,128,1,float16,float16,0,0.5041103839874268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,1,128,1,float16,float16,0,0.19678879976272584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,8,128,1,float16,fp8,0,0.4877151966094971
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,8,128,1,fp8,fp8,0,0.48661441802978517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,32,128,1,float16,float16,0,0.5032959938049316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,32,128,1,float16,fp8,0,0.4850783824920654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,32,128,1,fp8,fp8,0,0.48352479934692383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,1,128,1,float16,fp8,0,0.19607839584350586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,1,128,1,fp8,fp8,0,0.19603680372238158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,2,128,1,float16,float16,0,0.2050447940826416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,2,128,1,float16,fp8,0,0.20353279113769532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,2,128,1,fp8,fp8,0,0.20346879959106445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,4,128,1,float16,float16,0,0.2303663969039917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,8,128,1,fp8,fp8,0,0.2611952066421509
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,4,128,1,float16,fp8,0,0.22665600776672362
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,4,128,1,fp8,fp8,0,0.22638239860534667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,8,128,1,float16,float16,0,0.2614799976348877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,8,128,1,float16,fp8,0,0.2560895919799805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,32,128,1,float16,float16,0,0.26512000560760496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,32,128,1,float16,fp8,0,0.24854719638824463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,2,128,1,float16,fp8,0,0.11601120233535767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,1,128,1,float16,float16,0,0.11131360530853271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,32,128,1,fp8,fp8,0,0.25484960079193114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,1,128,1,float16,fp8,0,0.11087520122528076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,1,128,1,fp8,fp8,0,0.11188640594482421
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,2,128,1,float16,float16,0,0.11484800577163697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,8,128,1,float16,fp8,0,0.14268159866333008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,2,128,1,fp8,fp8,0,0.11842559576034546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,4,128,1,float16,float16,0,0.1258255958557129
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,4,128,1,float16,fp8,0,0.1251904010772705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,4,128,1,fp8,fp8,0,0.1242624044418335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,8,128,1,float16,float16,0,0.1459920048713684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,8,128,1,fp8,fp8,0,0.1418768048286438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,32,128,1,float16,float16,0,0.14319519996643065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,32,128,1,float16,fp8,0,0.13721760511398315
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,32,128,1,fp8,fp8,0,0.13768160343170166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,1,128,1,float16,float16,0,0.06360960006713867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,1,128,1,float16,fp8,0,0.06289119720458984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,4,128,1,float16,fp8,0,0.06932640075683594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,1,128,1,fp8,fp8,0,0.061680001020431516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,2,128,1,float16,float16,0,0.06615200042724609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,2,128,1,float16,fp8,0,0.06136639714241028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,2,128,1,fp8,fp8,0,0.06228960156440735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,4,128,1,float16,float16,0,0.07280640006065368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,4,128,1,fp8,fp8,0,0.06987360119819641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,8,128,1,float16,float16,0,0.08192319869995117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,8,128,1,float16,fp8,0,0.07758560180664062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,8,128,1,fp8,fp8,0,0.07670720219612122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,32,128,1,float16,float16,0,0.08112000226974488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,32,128,1,float16,fp8,0,0.06955519914627076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,32,128,1,fp8,fp8,0,0.07131519913673401
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,1,128,1,float16,float16,0,0.0392192006111145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,1,128,1,float16,fp8,0,0.04023520052433014
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,1,128,1,fp8,fp8,0,0.04043999910354614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,2,128,1,float16,float16,0,0.0399071991443634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,2,128,1,float16,fp8,0,0.04092159867286682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,2,128,1,fp8,fp8,0,0.04033919870853424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,4,128,1,float16,float16,0,0.040931200981140135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,4,128,1,float16,fp8,0,0.04192480146884918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,4,128,1,fp8,fp8,0,0.04156000018119812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,8,128,1,float16,float16,0,0.04635359942913055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,8,128,1,float16,fp8,0,0.04585599899291992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,8,128,1,fp8,fp8,0,0.04605759978294373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,1,128,1,float16,float16,0,2.0225744247436523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,1,128,1,float16,fp8,0,2.0635536193847654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,1,128,1,fp8,fp8,0,2.0435440063476564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,2,128,1,float16,float16,0,2.1771215438842773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,2,128,1,float16,fp8,0,2.2032928466796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,2,128,1,fp8,fp8,0,2.1873552322387697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,4,128,1,float16,float16,0,2.4993343353271484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,4,128,1,float16,fp8,0,2.494607925415039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,4,128,1,fp8,fp8,0,2.481185531616211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,8,128,1,float16,float16,0,3.150494384765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,8,128,1,float16,fp8,0,3.0309024810791017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,8,128,1,fp8,fp8,0,3.0320175170898436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,1,128,1,float16,float16,0,1.0171152114868165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,32,128,1,float16,float16,0,3.398110580444336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,1,128,1,float16,fp8,0,1.0318400382995605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,32,128,1,float16,fp8,0,3.2940704345703127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,32,128,1,fp8,fp8,0,3.307585525512695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,1,128,1,fp8,fp8,0,1.0352928161621093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,2,128,1,float16,float16,0,1.0948240280151367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,2,128,1,float16,fp8,0,1.099619197845459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,2,128,1,fp8,fp8,0,1.116100788116455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,4,128,1,float16,float16,0,1.255072021484375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,4,128,1,float16,fp8,0,1.2385760307312013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,4,128,1,fp8,fp8,0,1.2441760063171388
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,8,128,1,float16,float16,0,1.569161605834961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,1,128,1,float16,float16,0,0.5204080104827881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,8,128,1,float16,fp8,0,1.5290335655212401
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,8,128,1,fp8,fp8,0,1.5263952255249023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,1,128,1,float16,fp8,0,0.5281375885009766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,32,128,1,float16,float16,0,1.6860528945922852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,32,128,1,float16,fp8,0,1.6910335540771484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,1,128,1,fp8,fp8,0,0.5287216186523438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,2,128,1,float16,float16,0,0.5583263874053955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,32,128,1,fp8,fp8,0,1.6836048126220704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,2,128,1,float16,fp8,0,0.5639135837554932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,2,128,1,fp8,fp8,0,0.5606863975524903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,4,128,1,float16,float16,0,0.6370880126953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,4,128,1,float16,fp8,0,0.6345551967620849
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,4,128,1,fp8,fp8,0,0.6376976013183594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,8,128,1,float16,float16,0,0.7955152034759522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,8,128,1,float16,fp8,0,0.7849215984344482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,8,128,1,fp8,fp8,0,0.7845407962799072
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,1,128,1,float16,float16,0,0.275979208946228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,32,128,1,float16,float16,0,0.8586064338684082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,32,128,1,float16,fp8,0,0.8357343673706055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,1,128,1,float16,fp8,0,0.276911997795105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,1,128,1,fp8,fp8,0,0.27596158981323243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,32,128,1,fp8,fp8,0,0.8392576217651367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,2,128,1,float16,float16,0,0.2965152025222778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,2,128,1,float16,fp8,0,0.29603679180145265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,8,128,1,float16,float16,0,0.4105072021484375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,2,128,1,fp8,fp8,0,0.29650239944458007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,4,128,1,float16,float16,0,0.32944159507751464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,4,128,1,float16,fp8,0,0.3273200035095215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,4,128,1,fp8,fp8,0,0.32993121147155763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,8,128,1,float16,fp8,0,0.39706718921661377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,8,128,1,fp8,fp8,0,0.4032127857208252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,32,128,1,float16,float16,0,0.44896960258483887
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,32,128,1,float16,fp8,0,0.43257918357849123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,2,128,1,float16,float16,0,0.1576751947402954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,2,128,1,float16,fp8,0,0.1590767979621887
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,1,128,1,float16,float16,0,0.15203360319137574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,32,128,1,fp8,fp8,0,0.4371200084686279
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,1,128,1,float16,fp8,0,0.15036000013351442
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,1,128,1,fp8,fp8,0,0.15093120336532592
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,2,128,1,fp8,fp8,0,0.15911840200424193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,4,128,1,float16,float16,0,0.17764960527420043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,4,128,1,float16,fp8,0,0.17592480182647705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,4,128,1,fp8,fp8,0,0.17731039524078368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,8,128,1,float16,float16,0,0.2205104112625122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,8,128,1,float16,fp8,0,0.21175839900970458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,8,128,1,fp8,fp8,0,0.21023519039154054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,32,128,1,float16,float16,0,0.23699839115142823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,32,128,1,float16,fp8,0,0.22899839878082276
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,32,128,1,fp8,fp8,0,0.23182399272918702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,2,128,1,fp8,fp8,0,0.09210079908370972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,1,128,1,float16,float16,0,0.08610079884529113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,1,128,1,float16,fp8,0,0.08695520162582397
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,1,128,1,fp8,fp8,0,0.08805599808692932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,2,128,1,float16,float16,0,0.09201440215110779
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,2,128,1,float16,fp8,0,0.09239360094070434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,8,128,1,fp8,fp8,0,0.11936960220336915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,32,128,1,float16,float16,0,0.12994400262832642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,4,128,1,float16,float16,0,0.10088160037994384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,4,128,1,float16,fp8,0,0.10107519626617431
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,32,128,1,fp8,fp8,0,0.12776000499725343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,4,128,1,fp8,fp8,0,0.1001520037651062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,8,128,1,float16,float16,0,0.12043199539184571
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,8,128,1,float16,fp8,0,0.1186911940574646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,32,128,1,float16,fp8,0,0.1275231957435608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,1,128,1,float16,float16,0,0.051923197507858274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,1,128,1,float16,fp8,0,0.04932479858398438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,1,128,1,fp8,fp8,0,0.05028480291366577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,2,128,1,float16,float16,0,0.05308640003204346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,2,128,1,float16,fp8,0,0.05092960000038147
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,2,128,1,fp8,fp8,0,0.05026400089263916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,4,128,1,float16,float16,0,0.06109920144081116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,4,128,1,float16,fp8,0,0.05626559853553772
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,4,128,1,fp8,fp8,0,0.05734879970550537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,8,128,1,float16,float16,0,0.06984480023384095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,8,128,1,float16,fp8,0,0.06459040045738221
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,8,128,1,fp8,fp8,0,0.06554880142211914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,32,128,1,float16,float16,0,0.07624319791793824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,32,128,1,float16,fp8,0,0.06934239864349365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,32,128,1,fp8,fp8,0,0.06772639751434326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,1,128,1,float16,float16,0,0.03330399990081787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,1,128,1,float16,fp8,0,0.03463039994239807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,1,128,1,fp8,fp8,0,0.03447999954223633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,2,128,1,float16,float16,0,0.033497598767280576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,2,128,1,float16,fp8,0,0.034990400075912476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,2,128,1,fp8,fp8,0,0.03523040115833283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,4,128,1,float16,float16,0,0.034267199039459226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,4,128,1,float16,fp8,0,0.035867199301719666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,4,128,1,fp8,fp8,0,0.035489600896835324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,8,128,1,float16,float16,0,0.03912000060081482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,8,128,1,float16,fp8,0,0.04022560119628906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,8,128,1,fp8,fp8,0,0.039980798959732056
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,32,128,1,float16,float16,0,0.04167200028896332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,32,128,1,float16,fp8,0,0.0428272008895874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,32,128,1,fp8,fp8,0,0.0426831990480423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,1,128,1,float16,float16,0,0.028276801109313965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,1,128,1,float16,fp8,0,0.028977599740028382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,1,128,1,fp8,fp8,0,0.029187199473381043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,2,128,1,float16,float16,0,0.028571200370788575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,2,128,1,float16,fp8,0,0.029104000329971312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,2,128,1,fp8,fp8,0,0.030078399181365966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,4,128,1,float16,float16,0,0.028676798939704894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,4,128,1,float16,fp8,0,0.029628801345825195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,4,128,1,fp8,fp8,0,0.029836800694465638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,8,128,1,float16,float16,0,0.02905600070953369
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,8,128,1,float16,fp8,0,0.030187198519706727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,8,128,1,fp8,fp8,0,0.030206400156021117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,1,128,1,float16,float16,0,0.8161952018737793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,1,128,1,float16,fp8,0,0.8515263557434082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,1,128,1,fp8,fp8,0,0.8561967849731446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,2,128,1,float16,float16,0,0.8996479988098145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,2,128,1,float16,fp8,0,0.928923225402832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,2,128,1,fp8,fp8,0,0.9221296310424805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,4,128,1,float16,float16,0,1.0473520278930664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,4,128,1,float16,fp8,0,1.0665184020996095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,4,128,1,fp8,fp8,0,1.0666288375854491
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,8,128,1,float16,float16,0,1.3581248283386231
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,8,128,1,float16,fp8,0,1.353329563140869
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,8,128,1,fp8,fp8,0,1.3602864265441894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,1,128,1,float16,fp8,0,0.43837437629699705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,1,128,1,float16,float16,0,0.4230319976806641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,32,128,1,float16,float16,0,1.6504287719726562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,32,128,1,float16,fp8,0,1.5584943771362305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,32,128,1,fp8,fp8,0,1.5570976257324218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,2,128,1,float16,float16,0,0.4598896026611328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,2,128,1,float16,fp8,0,0.4728975772857666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,2,128,1,fp8,fp8,0,0.4733151912689209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,1,128,1,fp8,fp8,0,0.43873119354248047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,4,128,1,float16,float16,0,0.5375648021697998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,4,128,1,float16,fp8,0,0.5450880050659179
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,4,128,1,fp8,fp8,0,0.5449391841888428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,1,128,1,float16,float16,0,0.22301120758056642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,8,128,1,float16,float16,0,0.6907120227813721
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,8,128,1,fp8,fp8,0,0.6848559856414795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,8,128,1,float16,fp8,0,0.6873248100280762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,32,128,1,float16,float16,0,0.8271856307983398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,32,128,1,float16,fp8,0,0.792625617980957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,32,128,1,fp8,fp8,0,0.7921584129333497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,1,128,1,float16,fp8,0,0.23192639350891114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,1,128,1,fp8,fp8,0,0.2320512056350708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,2,128,1,float16,float16,0,0.24262399673461915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,2,128,1,float16,fp8,0,0.2494879961013794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,2,128,1,fp8,fp8,0,0.250273609161377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,4,128,1,float16,float16,0,0.27917919158935545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,4,128,1,float16,fp8,0,0.284934401512146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,4,128,1,fp8,fp8,0,0.2845423936843872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,8,128,1,float16,float16,0,0.35784640312194826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,8,128,1,float16,fp8,0,0.3545439958572388
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,8,128,1,fp8,fp8,0,0.35461599826812745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,1,128,1,fp8,fp8,0,0.12914400100708007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,32,128,1,float16,float16,0,0.4277632236480713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,32,128,1,float16,fp8,0,0.40808639526367185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,1,128,1,float16,float16,0,0.12503839731216432
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,32,128,1,fp8,fp8,0,0.4065392017364502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,1,128,1,float16,fp8,0,0.12927039861679077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,2,128,1,float16,float16,0,0.13288960456848145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,2,128,1,float16,fp8,0,0.13657439947128297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,2,128,1,fp8,fp8,0,0.1367840051651001
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,4,128,1,float16,float16,0,0.15202080011367797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,4,128,1,float16,fp8,0,0.15471999645233153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,4,128,1,fp8,fp8,0,0.15469599962234498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,8,128,1,float16,float16,0,0.1927296042442322
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,8,128,1,float16,fp8,0,0.18988800048828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,8,128,1,fp8,fp8,0,0.18930399417877197
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,32,128,1,float16,float16,0,0.22709760665893555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,32,128,1,float16,fp8,0,0.21517438888549806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,32,128,1,fp8,fp8,0,0.2159424066543579
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,2,128,1,float16,fp8,0,0.08097440004348755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,1,128,1,float16,float16,0,0.07410719990730286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,1,128,1,float16,fp8,0,0.07658560276031494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,1,128,1,fp8,fp8,0,0.07578240036964416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,2,128,1,float16,float16,0,0.08043040037155151
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,2,128,1,fp8,fp8,0,0.08076800107955932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,4,128,1,float16,float16,0,0.08769440054893493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,4,128,1,float16,fp8,0,0.08886719942092895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,4,128,1,fp8,fp8,0,0.08915519714355469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,8,128,1,float16,float16,0,0.10882400274276734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,8,128,1,float16,fp8,0,0.10736000537872314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,8,128,1,fp8,fp8,0,0.10653280019760132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,32,128,1,float16,float16,0,0.12519359588623047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,32,128,1,float16,fp8,0,0.11963679790496826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,32,128,1,fp8,fp8,0,0.11938079595565795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,1,128,1,float16,float16,0,0.04611839950084686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,1,128,1,float16,fp8,0,0.04472480118274689
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,1,128,1,fp8,fp8,0,0.044310399889945985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,2,128,1,float16,float16,0,0.048583999276161194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,2,128,1,float16,fp8,0,0.0453792005777359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,2,128,1,fp8,fp8,0,0.04589439928531647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,4,128,1,float16,float16,0,0.05558879971504212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,4,128,1,float16,fp8,0,0.051571202278137204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,4,128,1,fp8,fp8,0,0.05127840042114258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,32,128,1,fp8,fp8,0,0.06290879845619202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,8,128,1,float16,float16,0,0.06424319744110107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,1,128,1,float16,fp8,0,0.03270240128040314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,8,128,1,float16,fp8,0,0.05998079776763916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,8,128,1,fp8,fp8,0,0.059140801429748535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,32,128,1,float16,float16,0,0.07258080244064331
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,32,128,1,float16,fp8,0,0.06248160004615784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,1,128,1,float16,float16,0,0.0316895991563797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,1,128,1,fp8,fp8,0,0.03235679864883423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,2,128,1,float16,float16,0,0.031508800387382505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,2,128,1,float16,fp8,0,0.032888001203536986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,2,128,1,fp8,fp8,0,0.03275200128555298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,4,128,1,float16,float16,0,0.032576000690460204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,4,128,1,float16,fp8,0,0.03364799916744232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,4,128,1,fp8,fp8,0,0.0336432009935379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,8,128,1,float16,float16,0,0.037940800189971924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,8,128,1,float16,fp8,0,0.03829280138015747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,8,128,1,fp8,fp8,0,0.038176000118255615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,2,128,1,float16,float16,0,0.02547520101070404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,32,128,1,float16,float16,0,0.03951199948787689
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,32,128,1,float16,fp8,0,0.03831200003623962
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,32,128,1,fp8,fp8,0,0.03856480121612549
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,1,128,1,float16,float16,0,0.025167998671531678
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,1,128,1,float16,fp8,0,0.02609440088272095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,1,128,1,fp8,fp8,0,0.026251199841499328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,2,128,1,float16,fp8,0,0.02629440128803253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,2,128,1,fp8,fp8,0,0.026240000128746034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,4,128,1,float16,float16,0,0.02617599964141846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,4,128,1,float16,fp8,0,0.026158401370048524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,4,128,1,fp8,fp8,0,0.02627040147781372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,8,128,1,float16,float16,0,0.026387199759483337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,8,128,1,float16,fp8,0,0.027003198862075806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,8,128,1,fp8,fp8,0,0.02720319926738739
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,32,128,1,float16,float16,0,0.02953760027885437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,32,128,1,float16,fp8,0,0.030268800258636475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,32,128,1,fp8,fp8,0,0.030209600925445557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,1,128,1,float16,float16,0,0.023705600202083586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,1,128,1,float16,fp8,0,0.025147199630737305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,1,128,1,fp8,fp8,0,0.024617600440979003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,2,128,1,float16,float16,0,0.023839999735355378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,2,128,1,float16,fp8,0,0.025080001354217528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,2,128,1,fp8,fp8,0,0.02505599856376648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,4,128,1,float16,float16,0,0.024366399645805357
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,4,128,1,float16,fp8,0,0.024846400320529937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,4,128,1,fp8,fp8,0,0.024700799584388734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,1,128,1,float16,float16,0,0.4103663921356201
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,8,128,1,float16,float16,0,0.024271999299526215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,8,128,1,float16,fp8,0,0.024977600574493407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,8,128,1,fp8,fp8,0,0.02491360008716583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,1,128,1,float16,fp8,0,0.43624320030212405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,1,128,1,fp8,fp8,0,0.44047842025756834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,2,128,1,float16,float16,0,0.44496002197265627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,2,128,1,float16,fp8,0,0.4743231773376465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,2,128,1,fp8,fp8,0,0.47517919540405273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,4,128,1,float16,float16,0,0.5185808181762696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,4,128,1,float16,fp8,0,0.5506175994873047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,4,128,1,fp8,fp8,0,0.5510591983795166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,8,128,1,float16,float16,0,0.6676640033721923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,8,128,1,float16,fp8,0,0.7031375885009765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,8,128,1,fp8,fp8,0,0.699622392654419
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,32,128,1,float16,float16,0,0.8322768211364746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,1,128,1,float16,float16,0,0.21445279121398925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,32,128,1,float16,fp8,0,0.7978079795837403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,1,128,1,float16,fp8,0,0.22836799621582032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,1,128,1,fp8,fp8,0,0.22782719135284424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,32,128,1,fp8,fp8,0,0.7978240013122558
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,2,128,1,float16,float16,0,0.23288800716400146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,2,128,1,float16,fp8,0,0.24939839839935302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,2,128,1,fp8,fp8,0,0.24818720817565917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,4,128,1,float16,float16,0,0.26952319145202636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,8,128,1,float16,fp8,0,0.3586352109909058
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,4,128,1,float16,fp8,0,0.2854399919509888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,4,128,1,fp8,fp8,0,0.2848671913146973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,32,128,1,float16,float16,0,0.4296304225921631
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,8,128,1,float16,float16,0,0.34607999324798583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,8,128,1,fp8,fp8,0,0.35858399868011476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,1,128,1,float16,float16,0,0.1208016037940979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,32,128,1,float16,fp8,0,0.4094223976135254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,32,128,1,fp8,fp8,0,0.41004481315612795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,1,128,1,float16,fp8,0,0.12833600044250487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,1,128,1,fp8,fp8,0,0.1288159966468811
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,2,128,1,float16,float16,0,0.12760800123214722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,2,128,1,float16,fp8,0,0.13682880401611328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,2,128,1,fp8,fp8,0,0.13658879995346068
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,4,128,1,float16,float16,0,0.1473904013633728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,4,128,1,float16,fp8,0,0.15501760244369506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,4,128,1,fp8,fp8,0,0.1549631953239441
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,32,128,1,float16,fp8,0,0.21468000411987304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,8,128,1,float16,float16,0,0.1848736047744751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,8,128,1,float16,fp8,0,0.19145760536193848
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,8,128,1,fp8,fp8,0,0.19171839952468872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,32,128,1,float16,float16,0,0.22811360359191896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,32,128,1,fp8,fp8,0,0.21490240097045898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,1,128,1,float16,float16,0,0.07156640291213989
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,1,128,1,float16,fp8,0,0.07387840151786804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,1,128,1,fp8,fp8,0,0.07417920231819153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,2,128,1,float16,float16,0,0.07770400047302246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,2,128,1,float16,fp8,0,0.07994400262832642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,2,128,1,fp8,fp8,0,0.07976800203323364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,4,128,1,float16,float16,0,0.08633120059967041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,4,128,1,float16,fp8,0,0.08732640147209167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,32,128,1,float16,float16,0,0.1249008059501648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,4,128,1,fp8,fp8,0,0.08707360029220582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,8,128,1,float16,float16,0,0.10589599609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,8,128,1,float16,fp8,0,0.10596959590911866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,8,128,1,fp8,fp8,0,0.10611040592193603
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,32,128,1,float16,fp8,0,0.11663039922714233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,32,128,1,fp8,fp8,0,0.11660319566726685
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,1,128,1,float16,float16,0,0.04264799952507019
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,1,128,1,float16,fp8,0,0.0414000004529953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,1,128,1,fp8,fp8,0,0.042361599206924436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,2,128,1,float16,float16,0,0.04477919936180115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,2,128,1,float16,fp8,0,0.04201599955558777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,2,128,1,fp8,fp8,0,0.042387199401855466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,4,128,1,float16,float16,0,0.052127999067306516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,4,128,1,float16,fp8,0,0.04803679883480072
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,4,128,1,fp8,fp8,0,0.04800159931182861
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,8,128,1,float16,float16,0,0.06260799765586852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,8,128,1,float16,fp8,0,0.05712800025939942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,8,128,1,fp8,fp8,0,0.05735359787940979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,32,128,1,float16,float16,0,0.07028480172157288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,32,128,1,float16,fp8,0,0.05897279977798462
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,32,128,1,fp8,fp8,0,0.0589136004447937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,1,128,1,float16,float16,0,0.02874239981174469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,1,128,1,float16,fp8,0,0.029870399832725526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,1,128,1,fp8,fp8,0,0.029841598868370057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,2,128,1,float16,float16,0,0.028963199257850646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,2,128,1,float16,fp8,0,0.030028799176216127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,2,128,1,fp8,fp8,0,0.029811200499534608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,4,128,1,float16,float16,0,0.029864001274108886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,4,128,1,float16,fp8,0,0.030857598781585692
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,4,128,1,fp8,fp8,0,0.03094240128993988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,1,128,1,float16,float16,0,0.02370239943265915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,8,128,1,float16,float16,0,0.03463520109653473
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,8,128,1,float16,fp8,0,0.03515360057353974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,8,128,1,fp8,fp8,0,0.03514559864997864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,32,128,1,float16,float16,0,0.037118399143218996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,2,128,1,fp8,fp8,0,0.02452480047941208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,32,128,1,float16,fp8,0,0.036617600917816163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,32,128,1,fp8,fp8,0,0.036817601323127745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,1,128,1,float16,fp8,0,0.024507200717926024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,1,128,1,fp8,fp8,0,0.024545599520206452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,2,128,1,float16,float16,0,0.023894399404525757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,2,128,1,float16,fp8,0,0.024545599520206452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,4,128,1,float16,float16,0,0.024052800238132478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,4,128,1,float16,fp8,0,0.024747200310230255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,4,128,1,fp8,fp8,0,0.024484799802303316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,8,128,1,float16,float16,0,0.025009599328041077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,8,128,1,float16,fp8,0,0.0255295991897583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,8,128,1,fp8,fp8,0,0.025417599081993102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,32,128,1,float16,float16,0,0.028043198585510253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,32,128,1,float16,fp8,0,0.028502398729324342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,32,128,1,fp8,fp8,0,0.028641599416732787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,1,128,1,float16,float16,0,0.022227199375629426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,1,128,1,float16,fp8,0,0.022988800704479218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,1,128,1,fp8,fp8,0,0.023051199316978455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,2,128,1,float16,float16,0,0.022225600481033326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,2,128,1,float16,fp8,0,0.022894400358200073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,2,128,1,fp8,fp8,0,0.0232464000582695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,4,128,1,float16,float16,0,0.022676800191402436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,4,128,1,float16,fp8,0,0.023086400330066682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,4,128,1,fp8,fp8,0,0.022961600124835967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,8,128,1,float16,float16,0,0.02263839989900589
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,8,128,1,float16,fp8,0,0.023123200237751006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,8,128,1,fp8,fp8,0,0.02327200025320053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,32,128,1,float16,float16,0,0.022699199616909027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,32,128,1,float16,fp8,0,0.02337760031223297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,32,128,1,fp8,fp8,0,0.02375999987125397
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,1,128,1,float16,float16,0,0.021430400013923646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,1,128,1,float16,fp8,0,0.0220768004655838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,1,128,1,fp8,fp8,0,0.02197439968585968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,2,128,1,float16,float16,0,0.021270400285720824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,2,128,1,float16,fp8,0,0.022431999444961548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,2,128,1,fp8,fp8,0,0.022286400198936462
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,4,128,1,float16,float16,0,0.02158239930868149
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,4,128,1,float16,fp8,0,0.02280319929122925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,4,128,1,fp8,fp8,0,0.02208160012960434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,8,128,1,float16,float16,0,0.02160159945487976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,8,128,1,float16,fp8,0,0.022540800273418427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,8,128,1,fp8,fp8,0,0.02205439954996109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,2,128,1,float16,float16,0,0.23243520259857178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,1,128,1,float16,float16,0,0.2153248071670532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,1,128,1,float16,fp8,0,0.22899041175842286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,1,128,1,fp8,fp8,0,0.22860479354858398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,2,128,1,float16,fp8,0,0.24796640872955322
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,2,128,1,fp8,fp8,0,0.24908800125122071
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,4,128,1,float16,float16,0,0.2727839946746826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,4,128,1,float16,fp8,0,0.28459839820861815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,4,128,1,fp8,fp8,0,0.2848543882369995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,8,128,1,float16,float16,0,0.35567359924316405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,8,128,1,float16,fp8,0,0.35727040767669677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,8,128,1,fp8,fp8,0,0.35871360301971433
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,32,128,1,float16,float16,0,0.47524161338806153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,1,128,1,float16,float16,0,0.11975359916687012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,32,128,1,float16,fp8,0,0.48540477752685546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,1,128,1,float16,fp8,0,0.12862399816513062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,2,128,1,fp8,fp8,0,0.1371392011642456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,32,128,1,fp8,fp8,0,0.48504319190979006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,1,128,1,fp8,fp8,0,0.12860640287399291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,2,128,1,float16,float16,0,0.1284656047821045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,2,128,1,float16,fp8,0,0.13777439594268798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,4,128,1,float16,float16,0,0.1481775999069214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,4,128,1,float16,fp8,0,0.155622398853302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,4,128,1,fp8,fp8,0,0.15564639568328859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,8,128,1,float16,float16,0,0.18732800483703613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,8,128,1,float16,fp8,0,0.19125280380249024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,8,128,1,fp8,fp8,0,0.1920143961906433
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,32,128,1,float16,float16,0,0.25064640045166015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,32,128,1,float16,fp8,0,0.25263679027557373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,1,128,1,float16,float16,0,0.07290719747543335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,32,128,1,fp8,fp8,0,0.2521984100341797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,1,128,1,float16,fp8,0,0.07467839717864991
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,1,128,1,fp8,fp8,0,0.07471039891242981
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,2,128,1,float16,float16,0,0.0785647988319397
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,2,128,1,float16,fp8,0,0.0808784008026123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,2,128,1,fp8,fp8,0,0.08107360005378723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,4,128,1,float16,float16,0,0.08696960210800171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,4,128,1,float16,fp8,0,0.08756160140037536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,4,128,1,fp8,fp8,0,0.08798239827156067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,8,128,1,float16,float16,0,0.10636639595031738
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,8,128,1,float16,fp8,0,0.10685440301895141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,8,128,1,fp8,fp8,0,0.1066480040550232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,32,128,1,float16,float16,0,0.13678560256958008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,32,128,1,float16,fp8,0,0.13509759902954102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,32,128,1,fp8,fp8,0,0.13505760431289673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,1,128,1,float16,float16,0,0.04189600050449371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,1,128,1,float16,fp8,0,0.04248960018157959
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,1,128,1,fp8,fp8,0,0.04166080057621002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,2,128,1,float16,float16,0,0.04466400146484375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,2,128,1,float16,fp8,0,0.04267680048942566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,2,128,1,fp8,fp8,0,0.042559999227523806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,4,128,1,float16,float16,0,0.0537280023097992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,4,128,1,float16,fp8,0,0.04737440049648285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,4,128,1,fp8,fp8,0,0.04867999851703644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,32,128,1,fp8,fp8,0,0.06768640279769897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,8,128,1,float16,float16,0,0.06199520230293274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,8,128,1,float16,fp8,0,0.0568336009979248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,8,128,1,fp8,fp8,0,0.058462399244308474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,32,128,1,float16,float16,0,0.07802720069885254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,32,128,1,float16,fp8,0,0.0674239993095398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,1,128,1,float16,float16,0,0.02834559977054596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,1,128,1,float16,fp8,0,0.030015999078750612
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,1,128,1,fp8,fp8,0,0.029951998591423036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,2,128,1,float16,float16,0,0.02847360074520111
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,2,128,1,float16,fp8,0,0.030048000812530517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,2,128,1,fp8,fp8,0,0.0298768013715744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,4,128,1,float16,float16,0,0.02963840067386627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,4,128,1,float16,fp8,0,0.030635198950767516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,4,128,1,fp8,fp8,0,0.030801600217819212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,8,128,1,float16,float16,0,0.03495199978351593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,8,128,1,float16,fp8,0,0.03490239977836609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,8,128,1,fp8,fp8,0,0.03508319854736328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,32,128,1,float16,float16,0,0.040084800124168395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,32,128,1,float16,fp8,0,0.04104160070419312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,32,128,1,fp8,fp8,0,0.04079039990901947
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,1,128,1,float16,float16,0,0.023708799481391908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,4,128,1,float16,fp8,0,0.024607999622821806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,1,128,1,float16,fp8,0,0.0247871994972229
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,1,128,1,fp8,fp8,0,0.0246288001537323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,2,128,1,float16,float16,0,0.023478400707244874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,2,128,1,float16,fp8,0,0.02471359968185425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,2,128,1,fp8,fp8,0,0.024560000002384185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,4,128,1,float16,float16,0,0.023956799507141115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,4,128,1,fp8,fp8,0,0.024676799774169922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,8,128,1,float16,float16,0,0.024663999676704407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,8,128,1,float16,fp8,0,0.02558720111846924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,8,128,1,fp8,fp8,0,0.025574401021003723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,32,128,1,float16,float16,0,0.027241599559783936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,32,128,1,float16,fp8,0,0.028190401196479798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,32,128,1,fp8,fp8,0,0.028147199749946596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,1,128,1,float16,float16,0,0.022390399873256684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,1,128,1,float16,fp8,0,0.02309280037879944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,1,128,1,fp8,fp8,0,0.02304159998893738
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,2,128,1,float16,float16,0,0.0221903994679451
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,2,128,1,float16,fp8,0,0.023177599906921385
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,2,128,1,fp8,fp8,0,0.02322079986333847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,4,128,1,float16,float16,0,0.02248319983482361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,4,128,1,float16,fp8,0,0.023558400571346283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,4,128,1,fp8,fp8,0,0.02284960001707077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,8,128,1,float16,float16,0,0.02239519953727722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,8,128,1,float16,fp8,0,0.02332960069179535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,8,128,1,fp8,fp8,0,0.023153600096702576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,32,128,1,float16,float16,0,0.02208320051431656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,32,128,1,float16,fp8,0,0.022865599393844603
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,32,128,1,fp8,fp8,0,0.023401600122451783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,1,128,1,float16,float16,0,0.02102559953927994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,1,128,1,float16,fp8,0,0.02229759991168976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,1,128,1,fp8,fp8,0,0.022065599262714387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,2,128,1,float16,float16,0,0.021324799954891206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,2,128,1,float16,fp8,0,0.022252799570560457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,2,128,1,fp8,fp8,0,0.02210720032453537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,4,128,1,float16,float16,0,0.021848000586032867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,4,128,1,float16,fp8,0,0.02208479940891266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,4,128,1,fp8,fp8,0,0.02237759977579117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,8,128,1,float16,float16,0,0.021454399824142455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,8,128,1,float16,fp8,0,0.022302399575710296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,8,128,1,fp8,fp8,0,0.022150400280952453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,32,128,1,float16,float16,0,0.021254399418830873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,32,128,1,float16,fp8,0,0.022431999444961548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,32,128,1,fp8,fp8,0,0.02181279957294464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,1,128,1,float16,float16,0,0.020640000700950623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,1,128,1,float16,fp8,0,0.0215488001704216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,1,128,1,fp8,fp8,0,0.021355199813842773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,2,128,1,float16,float16,0,0.02069759964942932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,2,128,1,float16,fp8,0,0.021516799926757812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,2,128,1,fp8,fp8,0,0.021844799816608428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,4,128,1,float16,float16,0,0.02114560008049011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,4,128,1,float16,fp8,0,0.021780799329280853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,4,128,1,fp8,fp8,0,0.02162880003452301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,8,128,1,float16,float16,0,0.021143999695777894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,8,128,1,float16,fp8,0,0.02170239984989166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,8,128,1,fp8,fp8,0,0.021671999990940095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,1,128,1,float16,float16,0,0.12097599506378173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,1,128,1,float16,fp8,0,0.13054879903793334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,1,128,1,fp8,fp8,0,0.13005280494689941
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,2,128,1,float16,float16,0,0.13014400005340576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,2,128,1,float16,fp8,0,0.13715519905090331
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,8,128,1,float16,float16,0,0.18591680526733398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,8,128,1,float16,fp8,0,0.19213600158691407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,2,128,1,fp8,fp8,0,0.13773280382156372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,4,128,1,float16,float16,0,0.14866240024566652
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,4,128,1,float16,fp8,0,0.15595040321350098
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,4,128,1,fp8,fp8,0,0.15677599906921386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,8,128,1,fp8,fp8,0,0.19219199419021607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,32,128,1,float16,float16,0,0.30714399814605714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,2,128,1,float16,float16,0,0.07863039970397949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,32,128,1,float16,fp8,0,0.32257440090179446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,1,128,1,float16,float16,0,0.07154560089111328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,32,128,1,fp8,fp8,0,0.32107679843902587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,1,128,1,float16,fp8,0,0.07548800110816956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,1,128,1,fp8,fp8,0,0.0757423996925354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,2,128,1,float16,fp8,0,0.08142399787902832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,2,128,1,fp8,fp8,0,0.08166400194168091
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,4,128,1,float16,float16,0,0.08726879954338074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,4,128,1,float16,fp8,0,0.0889952003955841
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,4,128,1,fp8,fp8,0,0.08870400190353393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,8,128,1,float16,float16,0,0.10570720434188843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,8,128,1,float16,fp8,0,0.10686240196228028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,8,128,1,fp8,fp8,0,0.1069375991821289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,32,128,1,float16,float16,0,0.16669440269470215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,32,128,1,float16,fp8,0,0.1720703959465027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,32,128,1,fp8,fp8,0,0.17276480197906494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,1,128,1,float16,float16,0,0.04306240081787109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,1,128,1,float16,fp8,0,0.04129120111465454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,1,128,1,fp8,fp8,0,0.04119519889354706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,2,128,1,float16,float16,0,0.046614399552345274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,2,128,1,float16,fp8,0,0.04193600118160248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,2,128,1,fp8,fp8,0,0.042710399627685545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,4,128,1,float16,float16,0,0.053041601181030275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,4,128,1,float16,fp8,0,0.048737600445747375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,4,128,1,fp8,fp8,0,0.04858080148696899
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,8,128,1,float16,float16,0,0.06356800198554993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,8,128,1,float16,fp8,0,0.05568479895591736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,8,128,1,fp8,fp8,0,0.05718240141868591
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,32,128,1,float16,float16,0,0.09308639764785767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,32,128,1,float16,fp8,0,0.08744959831237793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,32,128,1,fp8,fp8,0,0.08697280287742615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,1,128,1,float16,float16,0,0.02873600125312805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,1,128,1,float16,fp8,0,0.030035200715065002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,1,128,1,fp8,fp8,0,0.03020319938659668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,2,128,1,float16,float16,0,0.02884800136089325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,2,128,1,float16,fp8,0,0.030275198817253112
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,2,128,1,fp8,fp8,0,0.030270400643348693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,4,128,1,float16,float16,0,0.02982879877090454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,4,128,1,float16,fp8,0,0.03135519921779632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,4,128,1,fp8,fp8,0,0.03135519921779632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,8,128,1,float16,float16,0,0.034596800804138184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,8,128,1,float16,fp8,0,0.03543039858341217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,8,128,1,fp8,fp8,0,0.03533119857311249
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,32,128,1,float16,float16,0,0.0479200005531311
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,32,128,1,float16,fp8,0,0.04989120066165924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,32,128,1,fp8,fp8,0,0.04983200132846832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,1,128,1,float16,float16,0,0.023835200071334838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,1,128,1,float16,fp8,0,0.024508799612522125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,1,128,1,fp8,fp8,0,0.02465440034866333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,4,128,1,fp8,fp8,0,0.024728000164031982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,2,128,1,float16,float16,0,0.023665599524974823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,2,128,1,float16,fp8,0,0.024539199471473695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,2,128,1,fp8,fp8,0,0.02434239983558655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,4,128,1,float16,float16,0,0.023875199258327484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,4,128,1,float16,fp8,0,0.024984000623226164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,8,128,1,float16,float16,0,0.024667200446128846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,8,128,1,float16,fp8,0,0.025417599081993102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,8,128,1,fp8,fp8,0,0.025367999076843263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,32,128,1,float16,float16,0,0.03137759864330292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,32,128,1,float16,fp8,0,0.0326335996389389
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,32,128,1,fp8,fp8,0,0.03256320059299469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,1,128,1,float16,float16,0,0.022172799706459044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,1,128,1,float16,fp8,0,0.022835199534893037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,1,128,1,fp8,fp8,0,0.023337599635124207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,2,128,1,float16,float16,0,0.022404800355434417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,2,128,1,float16,fp8,0,0.02326720058917999
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,2,128,1,fp8,fp8,0,0.023255999386310577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,4,128,1,float16,float16,0,0.02239840030670166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,4,128,1,float16,fp8,0,0.022811199724674224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,4,128,1,fp8,fp8,0,0.02306720018386841
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,8,128,1,float16,float16,0,0.022843199968338012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,8,128,1,float16,fp8,0,0.023150399327278137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,8,128,1,fp8,fp8,0,0.023395200073719025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,32,128,1,float16,float16,0,0.022350400686264038
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,32,128,1,float16,fp8,0,0.023332799971103668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,32,128,1,fp8,fp8,0,0.022971199452877046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,1,128,1,float16,float16,0,0.02139520049095154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,1,128,1,float16,fp8,0,0.021955199539661407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,1,128,1,fp8,fp8,0,0.021825599670410156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,2,128,1,float16,float16,0,0.021305599808692934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,2,128,1,float16,fp8,0,0.02200320065021515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,2,128,1,fp8,fp8,0,0.022163200378417968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,4,128,1,float16,float16,0,0.0213919997215271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,4,128,1,float16,fp8,0,0.022142399847507478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,4,128,1,fp8,fp8,0,0.022103999555110932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,8,128,1,float16,float16,0,0.02141920030117035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,8,128,1,float16,fp8,0,0.022235199809074402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,8,128,1,fp8,fp8,0,0.022299200296401978
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,32,128,1,float16,float16,0,0.021217599511146545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,32,128,1,float16,fp8,0,0.0221791997551918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,32,128,1,fp8,fp8,0,0.022265599668025972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,1,128,1,float16,float16,0,0.02062560021877289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,1,128,1,float16,fp8,0,0.021436800062656403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,1,128,1,fp8,fp8,0,0.021481600403785706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,2,128,1,float16,float16,0,0.02095839977264404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,2,128,1,float16,fp8,0,0.021835200488567352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,8,128,1,fp8,fp8,0,0.0219200000166893
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,2,128,1,fp8,fp8,0,0.021512000262737273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,32,128,1,float16,float16,0,0.021201600134372712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,4,128,1,float16,float16,0,0.020772799849510193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,4,128,1,float16,fp8,0,0.021404799818992615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,4,128,1,fp8,fp8,0,0.021483199298381807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,8,128,1,float16,float16,0,0.021022400259971617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,8,128,1,float16,fp8,0,0.021663999557495116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,32,128,1,float16,fp8,0,0.021799999475479125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,32,128,1,fp8,fp8,0,0.021782399713993074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,1,128,1,float16,float16,0,0.020604799687862396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,1,128,1,float16,fp8,0,0.0213919997215271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,1,128,1,fp8,fp8,0,0.021161599457263945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,2,128,1,float16,float16,0,0.020532800257205962
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,2,128,1,float16,fp8,0,0.021241599321365358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,2,128,1,fp8,fp8,0,0.02143999934196472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,4,128,1,float16,float16,0,0.020644800364971162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,4,128,1,float16,fp8,0,0.021278400719165803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,4,128,1,fp8,fp8,0,0.021593600511550903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,8,128,1,float16,float16,0,0.02056480050086975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,8,128,1,float16,fp8,0,0.02128639966249466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,8,128,1,fp8,fp8,0,0.02133280038833618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,1,128,1,float16,float16,0,0.0742959976196289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,1,128,1,float16,fp8,0,0.07702400088310242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,1,128,1,fp8,fp8,0,0.07714239954948425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,2,128,1,float16,float16,0,0.07950239777565002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,8,128,1,float16,float16,0,0.13595199584960938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,2,128,1,float16,fp8,0,0.08313599824905396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,2,128,1,fp8,fp8,0,0.08268640041351319
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,4,128,1,float16,float16,0,0.08709440231323243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,4,128,1,float16,fp8,0,0.08974559903144837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,4,128,1,fp8,fp8,0,0.09016960263252258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,8,128,1,float16,fp8,0,0.14288480281829835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,1,128,1,float16,fp8,0,0.04223040044307709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,8,128,1,fp8,fp8,0,0.14266400337219237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,32,128,1,float16,float16,0,0.224236798286438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,32,128,1,float16,fp8,0,0.24641599655151367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,1,128,1,float16,float16,0,0.044305598735809325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,32,128,1,fp8,fp8,0,0.24471518993377686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,1,128,1,fp8,fp8,0,0.04408319890499115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,2,128,1,float16,float16,0,0.045449599623680115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,2,128,1,float16,fp8,0,0.04397920072078705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,2,128,1,fp8,fp8,0,0.04329279959201813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,4,128,1,float16,float16,0,0.054369598627090454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,32,128,1,float16,float16,0,0.12348639965057373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,4,128,1,float16,fp8,0,0.05044320225715637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,4,128,1,fp8,fp8,0,0.0494735985994339
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,8,128,1,float16,float16,0,0.07912160158157348
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,8,128,1,float16,fp8,0,0.0755631983280182
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,8,128,1,fp8,fp8,0,0.07558079957962036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,32,128,1,float16,fp8,0,0.12318880558013916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,32,128,1,fp8,fp8,0,0.12366880178451538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,1,128,1,float16,float16,0,0.02951039969921112
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,1,128,1,float16,fp8,0,0.030825600028038025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,1,128,1,fp8,fp8,0,0.03089120090007782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,2,128,1,float16,float16,0,0.029262399673461913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,2,128,1,float16,fp8,0,0.030807998776435853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,2,128,1,fp8,fp8,0,0.031065601110458373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,4,128,1,float16,float16,0,0.03038879930973053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,4,128,1,float16,fp8,0,0.031876799464225766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,4,128,1,fp8,fp8,0,0.03185279965400696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,8,128,1,float16,float16,0,0.04190079867839813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,8,128,1,float16,fp8,0,0.04408800005912781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,8,128,1,fp8,fp8,0,0.044740799069404605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,32,128,1,float16,float16,0,0.06261280179023743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,32,128,1,float16,fp8,0,0.06804800033569336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,32,128,1,fp8,fp8,0,0.06896799802780151
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,1,128,1,float16,float16,0,0.02420479953289032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,1,128,1,float16,fp8,0,0.025064000487327577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,1,128,1,fp8,fp8,0,0.02497279942035675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,2,128,1,float16,float16,0,0.024139200150966645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,2,128,1,float16,fp8,0,0.024987199902534486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,2,128,1,fp8,fp8,0,0.025033599138259886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,4,128,1,float16,float16,0,0.024676799774169922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,4,128,1,float16,fp8,0,0.025153601169586183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,4,128,1,fp8,fp8,0,0.025353598594665527
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,8,128,1,float16,float16,0,0.028414401412010192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,8,128,1,float16,fp8,0,0.03009119927883148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,8,128,1,fp8,fp8,0,0.030100798606872557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,32,128,1,float16,float16,0,0.03829759955406189
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,32,128,1,float16,fp8,0,0.04191839993000031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,32,128,1,fp8,fp8,0,0.04161440134048462
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,1,128,1,float16,float16,0,0.0220768004655838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,1,128,1,float16,fp8,0,0.023286400735378264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,1,128,1,fp8,fp8,0,0.02351360023021698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,2,128,1,float16,float16,0,0.022363199293613432
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,2,128,1,float16,fp8,0,0.023187200725078582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,2,128,1,fp8,fp8,0,0.023265600204467773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,4,128,1,float16,float16,0,0.022382399439811705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,4,128,1,float16,fp8,0,0.023054400086402894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,4,128,1,fp8,fp8,0,0.02294880002737045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,8,128,1,float16,float16,0,0.022500799596309663
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,8,128,1,float16,fp8,0,0.0233487993478775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,8,128,1,fp8,fp8,0,0.022947199642658234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,32,128,1,float16,float16,0,0.026001599431037904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,2,128,1,float16,fp8,0,0.021886399388313292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,32,128,1,float16,fp8,0,0.027878400683403016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,32,128,1,fp8,fp8,0,0.028091201186180116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,1,128,1,float16,float16,0,0.021430400013923646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,1,128,1,float16,fp8,0,0.02191839963197708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,1,128,1,fp8,fp8,0,0.022547200322151184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,2,128,1,float16,float16,0,0.021435199677944182
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,2,128,1,fp8,fp8,0,0.021904000639915468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,4,128,1,float16,float16,0,0.021396799385547637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,4,128,1,float16,fp8,0,0.022433599829673766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,4,128,1,fp8,fp8,0,0.022120000422000886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,8,128,1,float16,float16,0,0.021220800280570985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,8,128,1,float16,fp8,0,0.022168000042438508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,8,128,1,fp8,fp8,0,0.02208320051431656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,32,128,1,float16,float16,0,0.021457600593566894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,32,128,1,float16,fp8,0,0.02200479954481125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,32,128,1,fp8,fp8,0,0.022014400362968443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,1,128,1,float16,float16,0,0.020734399557113647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,1,128,1,float16,fp8,0,0.02150239944458008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,1,128,1,fp8,fp8,0,0.021540799736976625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,2,128,1,float16,float16,0,0.020764799416065217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,2,128,1,float16,fp8,0,0.021648000180721282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,2,128,1,fp8,fp8,0,0.021483199298381807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,4,128,1,float16,float16,0,0.020632000267505647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,4,128,1,float16,fp8,0,0.021296000480651854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,32,128,1,fp8,fp8,0,0.021663999557495116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,4,128,1,fp8,fp8,0,0.022118400037288665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,8,128,1,float16,float16,0,0.021022400259971617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,8,128,1,float16,fp8,0,0.02173759937286377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,8,128,1,fp8,fp8,0,0.02159679979085922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,32,128,1,float16,float16,0,0.021147200465202333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,32,128,1,float16,fp8,0,0.022097599506378175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,1,128,1,float16,float16,0,0.020656000077724456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,1,128,1,float16,fp8,0,0.021398399770259858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,1,128,1,fp8,fp8,0,0.021673600375652313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,2,128,1,float16,float16,0,0.020798400044441223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,2,128,1,float16,fp8,0,0.02139039933681488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,2,128,1,fp8,fp8,0,0.02128320038318634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,4,128,1,float16,float16,0,0.020838400721549986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,4,128,1,float16,fp8,0,0.021422399580478667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,4,128,1,fp8,fp8,0,0.02134079933166504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,8,128,1,float16,float16,0,0.020414400100708007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,8,128,1,float16,fp8,0,0.02138720005750656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,8,128,1,fp8,fp8,0,0.021367999911308288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,32,128,1,float16,float16,0,0.020751999318599702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,32,128,1,float16,fp8,0,0.02152319997549057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,32,128,1,fp8,fp8,0,0.021686400473117828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,1,128,1,float16,float16,0,0.02048480063676834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,1,128,1,float16,fp8,0,0.021134400367736818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,1,128,1,fp8,fp8,0,0.02128320038318634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,2,128,1,float16,float16,0,0.020559999346733093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,2,128,1,float16,fp8,0,0.021270400285720824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,2,128,1,fp8,fp8,0,0.02156960070133209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,4,128,1,float16,float16,0,0.02036159932613373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,4,128,1,float16,fp8,0,0.021430400013923646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,4,128,1,fp8,fp8,0,0.021404799818992615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,8,128,1,float16,float16,0,0.020444799959659577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,8,128,1,float16,fp8,0,0.021505600214004515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,8,128,1,fp8,fp8,0,0.0209647998213768
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,1,128,1,float16,fp8,0,18.72291717529297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,1,128,1,fp8,fp8,0,18.523175048828126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,2,128,1,float16,fp8,0,18.748818969726564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,2,128,1,fp8,fp8,0,18.813511657714844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,1,128,1,float16,float16,0,22.759014892578126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,2,128,1,float16,float16,0,22.689411926269532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,4,128,1,float16,float16,0,22.425674438476562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,4,128,1,float16,fp8,0,19.211683654785155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,24,128,1,float16,fp8,0,11.437004852294923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,24,128,1,fp8,fp8,0,11.465099334716797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,24,128,1,float16,float16,0,13.675178527832031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,1,128,1,float16,float16,0,11.343787384033202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,4,128,1,fp8,fp8,0,20.73145294189453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,8,128,1,float16,fp8,0,21.031474304199218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,8,128,1,fp8,fp8,0,21.145466613769532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,8,128,1,float16,float16,0,24.85272674560547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,1,128,1,float16,fp8,0,9.752267456054687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,1,128,1,fp8,fp8,0,9.883934020996094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,2,128,1,float16,fp8,0,10.71985092163086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,2,128,1,float16,float16,0,11.88349609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,2,128,1,fp8,fp8,0,9.626630401611328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,4,128,1,float16,fp8,0,9.807393646240234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,4,128,1,float16,float16,0,11.601105499267579
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,4,128,1,fp8,fp8,0,9.5550048828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,24,128,1,float16,fp8,0,5.506702423095703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,24,128,1,float16,float16,0,6.451718139648437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,8,128,1,float16,fp8,0,9.860854339599609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,8,128,1,float16,float16,0,11.98487548828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,24,128,1,fp8,fp8,0,5.509396743774414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,8,128,1,fp8,fp8,0,10.038113403320313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,1,128,1,float16,float16,0,5.459737777709961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,1,128,1,float16,fp8,0,4.650915145874023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,1,128,1,fp8,fp8,0,4.656777572631836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,2,128,1,float16,fp8,0,4.858441543579102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,2,128,1,float16,float16,0,5.826019287109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,2,128,1,fp8,fp8,0,4.747208023071289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,4,128,1,float16,float16,0,5.735681533813477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,4,128,1,float16,fp8,0,4.867827224731445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,4,128,1,fp8,fp8,0,4.796705627441407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,8,128,1,float16,float16,0,5.585801696777343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,24,128,1,float16,float16,0,3.1752527236938475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,24,128,1,float16,fp8,0,2.7710687637329103
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,8,128,1,float16,fp8,0,4.923904037475586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,8,128,1,fp8,fp8,0,4.946361541748047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,24,128,1,fp8,fp8,0,2.882671928405762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,1,128,1,float16,fp8,0,2.341923141479492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,1,128,1,float16,float16,0,3.033425521850586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,1,128,1,fp8,fp8,0,2.410665512084961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,2,128,1,float16,fp8,0,2.364279937744141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,2,128,1,float16,float16,0,2.8277536392211915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,2,128,1,fp8,fp8,0,2.363275146484375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,4,128,1,float16,fp8,0,2.412875175476074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,4,128,1,fp8,fp8,0,2.4073968887329102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,4,128,1,float16,float16,0,2.8374719619750977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,8,128,1,float16,float16,0,3.0132383346557616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,8,128,1,float16,fp8,0,2.5428367614746095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,8,128,1,fp8,fp8,0,2.512073516845703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,1,128,1,float16,fp8,0,11.695305633544923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,1,128,1,fp8,fp8,0,11.26278076171875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,2,128,1,float16,fp8,0,11.499513244628906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,2,128,1,fp8,fp8,0,11.25502700805664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,1,128,1,float16,float16,0,13.789869689941407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,4,128,1,float16,fp8,0,11.603943634033204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,2,128,1,float16,float16,0,13.812615966796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,4,128,1,float16,float16,0,14.252650451660156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,24,128,1,float16,fp8,0,6.611777496337891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,24,128,1,fp8,fp8,0,6.621211242675781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,24,128,1,float16,float16,0,7.798397064208984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,1,128,1,float16,float16,0,6.2846321105957035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,4,128,1,fp8,fp8,0,11.309423828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,8,128,1,float16,fp8,0,11.481292724609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,8,128,1,fp8,fp8,0,12.015735626220703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,8,128,1,float16,float16,0,13.994752502441406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,1,128,1,float16,fp8,0,5.474020767211914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,1,128,1,fp8,fp8,0,5.404803085327148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,2,128,1,float16,fp8,0,5.411017608642578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,2,128,1,float16,float16,0,6.774868774414062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,2,128,1,fp8,fp8,0,5.403787231445312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,4,128,1,float16,float16,0,7.012057495117188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,4,128,1,float16,fp8,0,6.019963073730469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,4,128,1,fp8,fp8,0,5.621051025390625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,24,128,1,float16,fp8,0,3.3158111572265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,24,128,1,float16,float16,0,3.8879119873046877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,8,128,1,float16,fp8,0,5.728350448608398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,8,128,1,float16,float16,0,7.259019470214843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,8,128,1,fp8,fp8,0,5.704609680175781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,1,128,1,float16,fp8,0,2.6883216857910157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,24,128,1,fp8,fp8,0,3.4236351013183595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,1,128,1,fp8,fp8,0,2.6747247695922853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,1,128,1,float16,float16,0,3.757396697998047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,2,128,1,float16,fp8,0,2.713387107849121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,2,128,1,float16,float16,0,3.204060745239258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,2,128,1,fp8,fp8,0,2.768878364562988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,4,128,1,float16,float16,0,3.242510223388672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,4,128,1,float16,fp8,0,2.778179168701172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,4,128,1,fp8,fp8,0,2.8338895797729493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,8,128,1,float16,float16,0,3.3056350708007813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,8,128,1,float16,fp8,0,2.8699087142944335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,24,128,1,float16,float16,0,1.8792720794677735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,8,128,1,fp8,fp8,0,2.962433624267578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,24,128,1,float16,fp8,0,1.8540752410888672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,24,128,1,fp8,fp8,0,1.6829168319702148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,1,128,1,float16,float16,0,1.4934320449829102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,1,128,1,float16,fp8,0,1.4626928329467774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,1,128,1,fp8,fp8,0,1.3469200134277344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,2,128,1,float16,fp8,0,1.3770319938659668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,2,128,1,float16,float16,0,1.5572624206542969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,2,128,1,fp8,fp8,0,1.4383440017700195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,4,128,1,float16,float16,0,1.5221440315246582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,4,128,1,float16,fp8,0,1.5533007621765136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,4,128,1,fp8,fp8,0,1.4177248001098632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,8,128,1,float16,float16,0,1.5703248023986816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,8,128,1,float16,fp8,0,1.4488991737365722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,8,128,1,fp8,fp8,0,1.7338943481445312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,1,128,1,float16,fp8,0,7.657220458984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,1,128,1,fp8,fp8,0,7.815975952148437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,2,128,1,float16,fp8,0,7.878105926513672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,1,128,1,float16,float16,0,9.463285064697265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,2,128,1,float16,float16,0,9.052716827392578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,2,128,1,fp8,fp8,0,8.012210845947266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,4,128,1,float16,fp8,0,7.804068756103516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,4,128,1,float16,float16,0,9.753562927246094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,24,128,1,float16,fp8,0,4.800065612792968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,24,128,1,fp8,fp8,0,4.7878974914550785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,24,128,1,float16,float16,0,5.533892822265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,1,128,1,float16,float16,0,4.5313873291015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,4,128,1,fp8,fp8,0,7.960094451904297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,8,128,1,float16,fp8,0,8.389631652832032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,8,128,1,fp8,fp8,0,8.174336242675782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,8,128,1,float16,float16,0,9.716129302978516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,1,128,1,float16,fp8,0,3.7834144592285157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,1,128,1,fp8,fp8,0,3.7562255859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,2,128,1,float16,fp8,0,3.946131134033203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,2,128,1,float16,float16,0,4.483204650878906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,2,128,1,fp8,fp8,0,3.9152225494384765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,4,128,1,float16,fp8,0,3.923068618774414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,4,128,1,float16,float16,0,4.8179889678955075
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,4,128,1,fp8,fp8,0,3.9623985290527344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,24,128,1,float16,float16,0,2.7705888748168945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,8,128,1,float16,fp8,0,4.071523284912109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,24,128,1,float16,fp8,0,2.4344831466674806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,8,128,1,float16,float16,0,4.929425430297852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,8,128,1,fp8,fp8,0,4.082129669189453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,24,128,1,fp8,fp8,0,2.4158912658691407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,1,128,1,float16,fp8,0,1.9033456802368165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,1,128,1,float16,float16,0,2.183393669128418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,1,128,1,fp8,fp8,0,1.9307039260864258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,2,128,1,float16,fp8,0,1.9220111846923829
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,2,128,1,float16,float16,0,2.1544240951538085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,2,128,1,fp8,fp8,0,2.089299201965332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,4,128,1,float16,float16,0,2.2464223861694337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,4,128,1,float16,fp8,0,1.953019142150879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,4,128,1,fp8,fp8,0,1.967870330810547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,24,128,1,float16,float16,0,1.5066911697387695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,8,128,1,float16,float16,0,2.3210432052612306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,24,128,1,float16,fp8,0,1.2428560256958008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,8,128,1,fp8,fp8,0,2.0518543243408205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,8,128,1,float16,fp8,0,2.170822334289551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,1,128,1,float16,float16,0,1.0214032173156737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,1,128,1,float16,fp8,0,0.9569519996643067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,24,128,1,fp8,fp8,0,1.322164821624756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,1,128,1,fp8,fp8,0,1.0504847526550294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,2,128,1,float16,float16,0,1.0597920417785645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,2,128,1,float16,fp8,0,0.9692255973815918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,2,128,1,fp8,fp8,0,0.9736224174499511
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,4,128,1,float16,float16,0,1.0787599563598633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,4,128,1,float16,fp8,0,1.0374896049499511
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,4,128,1,fp8,fp8,0,0.9928239822387696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,8,128,1,float16,float16,0,1.1310864448547364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,8,128,1,float16,fp8,0,1.072760009765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,8,128,1,fp8,fp8,0,1.0775967597961427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,1,128,1,float16,fp8,0,10.080712127685548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,1,128,1,fp8,fp8,0,10.475889587402344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,2,128,1,float16,fp8,0,10.26328353881836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,2,128,1,fp8,fp8,0,10.389180755615234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,1,128,1,float16,float16,0,11.622176361083984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,4,128,1,float16,fp8,0,10.657736206054688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,2,128,1,float16,float16,0,12.077735900878906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,4,128,1,float16,float16,0,12.553511810302734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,24,128,1,float16,fp8,0,6.554057312011719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,24,128,1,fp8,fp8,0,6.588314819335937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,1,128,1,float16,float16,0,5.868555068969727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,24,128,1,float16,float16,0,7.700782775878906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,4,128,1,fp8,fp8,0,10.461801910400391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,8,128,1,float16,fp8,0,10.887179565429687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,8,128,1,fp8,fp8,0,11.336231994628907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,8,128,1,float16,float16,0,12.793787384033203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,1,128,1,float16,fp8,0,4.983832168579101
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,1,128,1,fp8,fp8,0,4.941774368286133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,2,128,1,float16,fp8,0,5.009329605102539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,2,128,1,float16,float16,0,5.900279998779297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,2,128,1,fp8,fp8,0,5.118694305419922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,4,128,1,float16,fp8,0,5.360579299926758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,4,128,1,float16,float16,0,6.6997215270996096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,4,128,1,fp8,fp8,0,5.22664794921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,8,128,1,float16,fp8,0,5.411932754516601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,24,128,1,float16,float16,0,3.7466785430908205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,8,128,1,float16,float16,0,6.982465362548828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,8,128,1,fp8,fp8,0,5.53853759765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,24,128,1,float16,fp8,0,3.3040847778320312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,24,128,1,fp8,fp8,0,3.3458415985107424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,1,128,1,float16,float16,0,3.0944080352783203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,1,128,1,float16,fp8,0,2.444254493713379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,1,128,1,fp8,fp8,0,2.640067291259766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,2,128,1,float16,float16,0,2.872702407836914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,2,128,1,float16,fp8,0,2.5375295639038087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,2,128,1,fp8,fp8,0,2.6104591369628904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,4,128,1,float16,float16,0,2.9227792739868166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,4,128,1,float16,fp8,0,2.8613487243652345
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,4,128,1,fp8,fp8,0,2.5863983154296877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,8,128,1,float16,float16,0,3.2581710815429688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,24,128,1,float16,float16,0,1.8180591583251953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,8,128,1,float16,fp8,0,2.78875675201416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,24,128,1,float16,fp8,0,1.6716047286987306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,8,128,1,fp8,fp8,0,2.7626176834106446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,1,128,1,float16,fp8,0,1.2447967529296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,1,128,1,float16,float16,0,1.6002416610717773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,24,128,1,fp8,fp8,0,1.8302623748779296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,1,128,1,fp8,fp8,0,1.3230863571166993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,2,128,1,float16,float16,0,1.3704416275024414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,2,128,1,float16,fp8,0,1.2679231643676758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,2,128,1,fp8,fp8,0,1.2583711624145508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,4,128,1,float16,float16,0,1.425220775604248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,4,128,1,float16,fp8,0,1.305241584777832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,4,128,1,fp8,fp8,0,1.4511391639709472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,24,128,1,float16,float16,0,0.9058639526367187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,8,128,1,float16,float16,0,1.5135135650634766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,24,128,1,float16,fp8,0,0.8496303558349609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,8,128,1,float16,fp8,0,1.3896800041198731
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,8,128,1,fp8,fp8,0,1.4653360366821289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,1,128,1,float16,float16,0,0.6740223884582519
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,24,128,1,fp8,fp8,0,0.9338224411010743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,1,128,1,float16,fp8,0,0.6658736228942871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,1,128,1,fp8,fp8,0,0.6770256042480469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,2,128,1,float16,float16,0,0.7394559860229493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,2,128,1,float16,fp8,0,0.6434192180633544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,2,128,1,fp8,fp8,0,0.646289587020874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,4,128,1,float16,float16,0,0.7093488216400147
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,4,128,1,float16,fp8,0,0.6656703948974609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,4,128,1,fp8,fp8,0,0.6949471950531005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,8,128,1,float16,float16,0,0.754198408126831
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,8,128,1,float16,fp8,0,0.7175039768218994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,8,128,1,fp8,fp8,0,0.7638815879821778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,1,128,1,float16,fp8,0,5.717139053344726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,1,128,1,fp8,fp8,0,5.757046508789062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,2,128,1,float16,fp8,0,5.806092834472656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,1,128,1,float16,float16,0,6.9130607604980465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,2,128,1,fp8,fp8,0,5.880939102172851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,2,128,1,float16,float16,0,6.7687538146972654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,4,128,1,float16,fp8,0,6.039123153686523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,4,128,1,float16,float16,0,7.0451408386230465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,1,128,1,float16,float16,0,3.2600017547607423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,24,128,1,float16,fp8,0,4.137692642211914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,24,128,1,float16,float16,0,4.599894332885742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,24,128,1,fp8,fp8,0,4.067563247680664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,4,128,1,fp8,fp8,0,6.0167182922363285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,8,128,1,float16,fp8,0,6.435018920898438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,8,128,1,fp8,fp8,0,6.462614440917969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,8,128,1,float16,float16,0,7.570049285888672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,1,128,1,float16,fp8,0,2.900864028930664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,1,128,1,fp8,fp8,0,2.840563201904297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,2,128,1,float16,fp8,0,2.9000383377075196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,2,128,1,float16,float16,0,3.4027473449707033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,2,128,1,fp8,fp8,0,2.8993808746337892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,4,128,1,float16,fp8,0,3.0355167388916016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,4,128,1,float16,float16,0,3.425831985473633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,4,128,1,fp8,fp8,0,3.092758369445801
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,24,128,1,float16,fp8,0,2.0431455612182616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,24,128,1,float16,float16,0,2.3394031524658203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,8,128,1,float16,fp8,0,3.2170879364013674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,8,128,1,fp8,fp8,0,3.2314510345458984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,8,128,1,float16,float16,0,3.740033721923828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,1,128,1,float16,float16,0,1.6604192733764649
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,1,128,1,float16,fp8,0,1.4401663780212401
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,24,128,1,fp8,fp8,0,2.059111976623535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,1,128,1,fp8,fp8,0,1.4233455657958984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,2,128,1,float16,float16,0,1.672217559814453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,2,128,1,float16,fp8,0,1.4716464042663575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,2,128,1,fp8,fp8,0,1.4744959831237794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,4,128,1,float16,float16,0,1.6860975265502929
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,4,128,1,float16,fp8,0,1.579075241088867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,4,128,1,fp8,fp8,0,1.5223695755004882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,8,128,1,float16,float16,0,1.841766357421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,8,128,1,float16,fp8,0,1.624777603149414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,24,128,1,float16,float16,0,1.1485247611999512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,24,128,1,float16,fp8,0,1.044206428527832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,1,128,1,float16,float16,0,0.7769951820373535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,24,128,1,fp8,fp8,0,1.1315744400024415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,8,128,1,fp8,fp8,0,1.7731279373168944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,1,128,1,float16,fp8,0,0.7341584205627442
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,1,128,1,fp8,fp8,0,0.8332207679748536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,2,128,1,float16,float16,0,0.7869184017181396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,2,128,1,float16,fp8,0,0.767790412902832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,2,128,1,fp8,fp8,0,0.7430384159088135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,4,128,1,float16,float16,0,0.8304544448852539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,4,128,1,float16,fp8,0,0.7808864116668701
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,4,128,1,fp8,fp8,0,0.7903984069824219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,8,128,1,float16,float16,0,0.9069231986999512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,8,128,1,float16,fp8,0,0.8805791854858398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,24,128,1,float16,float16,0,0.5832608222961426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,8,128,1,fp8,fp8,0,0.8463215827941895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,24,128,1,float16,fp8,0,0.5530640125274658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,24,128,1,fp8,fp8,0,0.5394815921783447
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,1,128,1,float16,float16,0,0.39630239009857177
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,1,128,1,float16,fp8,0,0.39060320854187014
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,1,128,1,fp8,fp8,0,0.39153759479522704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,2,128,1,float16,float16,0,0.4021952152252197
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,2,128,1,float16,fp8,0,0.3939807891845703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,2,128,1,fp8,fp8,0,0.40302720069885256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,4,128,1,float16,float16,0,0.41791839599609376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,4,128,1,float16,fp8,0,0.40234079360961916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,4,128,1,fp8,fp8,0,0.4118512153625488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,8,128,1,float16,float16,0,0.4553088188171387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,8,128,1,float16,fp8,0,0.4238895893096924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,8,128,1,fp8,fp8,0,0.4346000194549561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,1,128,1,float16,fp8,0,5.44415054321289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,1,128,1,fp8,fp8,0,5.427406311035156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,1,128,1,float16,float16,0,6.2172080993652346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,2,128,1,float16,fp8,0,5.5705921173095705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,2,128,1,fp8,fp8,0,5.520000076293945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,2,128,1,float16,float16,0,6.506398773193359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,4,128,1,float16,float16,0,6.7186126708984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,4,128,1,float16,fp8,0,5.8854209899902346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,24,128,1,float16,float16,0,4.777510452270508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,24,128,1,float16,fp8,0,4.359892654418945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,4,128,1,fp8,fp8,0,5.82805290222168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,24,128,1,fp8,fp8,0,4.449615859985352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,8,128,1,float16,fp8,0,6.398043060302735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,8,128,1,fp8,fp8,0,6.325439834594727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,8,128,1,float16,float16,0,7.4031982421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,1,128,1,float16,float16,0,2.9290464401245115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,1,128,1,float16,fp8,0,2.741967964172363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,1,128,1,fp8,fp8,0,2.7031312942504884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,2,128,1,float16,float16,0,3.1486560821533205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,2,128,1,float16,fp8,0,2.7687967300415037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,2,128,1,fp8,fp8,0,2.7740800857543944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,4,128,1,float16,float16,0,3.1697103500366213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,4,128,1,float16,fp8,0,2.967452812194824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,4,128,1,fp8,fp8,0,3.1956432342529295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,24,128,1,float16,float16,0,2.3605888366699217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,8,128,1,float16,fp8,0,3.2022430419921877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,24,128,1,float16,fp8,0,2.1839391708374025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,8,128,1,float16,float16,0,3.6088367462158204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,1,128,1,float16,float16,0,1.6886159896850585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,1,128,1,float16,fp8,0,1.3875311851501464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,24,128,1,fp8,fp8,0,2.383228874206543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,8,128,1,fp8,fp8,0,3.3908031463623045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,1,128,1,fp8,fp8,0,1.4130640029907227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,2,128,1,float16,float16,0,1.4980832099914552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,2,128,1,float16,fp8,0,1.395076847076416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,2,128,1,fp8,fp8,0,1.5122783660888672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,4,128,1,float16,fp8,0,1.4765727996826172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,4,128,1,float16,float16,0,1.6452255249023438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,4,128,1,fp8,fp8,0,1.478273582458496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,8,128,1,float16,float16,0,1.7218799591064453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,1,128,1,float16,float16,0,0.7617199897766114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,1,128,1,float16,fp8,0,0.6949295997619629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,24,128,1,float16,float16,0,1.193056011199951
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,8,128,1,float16,fp8,0,1.6461584091186523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,24,128,1,float16,fp8,0,1.150001621246338
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,24,128,1,fp8,fp8,0,1.1112848281860352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,8,128,1,fp8,fp8,0,1.7282447814941406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,1,128,1,fp8,fp8,0,0.7351808071136474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,2,128,1,float16,float16,0,0.7908624172210693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,2,128,1,float16,fp8,0,0.7249824047088623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,2,128,1,fp8,fp8,0,0.725873613357544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,4,128,1,float16,float16,0,0.7931392192840576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,4,128,1,float16,fp8,0,0.7520912170410157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,4,128,1,fp8,fp8,0,0.7692063808441162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,8,128,1,float16,float16,0,0.887553596496582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,24,128,1,float16,float16,0,0.6082992076873779
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,8,128,1,float16,fp8,0,0.8408111572265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,8,128,1,fp8,fp8,0,0.8440959930419922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,24,128,1,float16,fp8,0,0.579420804977417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,1,128,1,float16,float16,0,0.3757263898849487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,24,128,1,fp8,fp8,0,0.5800735950469971
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,1,128,1,float16,fp8,0,0.36851840019226073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,4,128,1,float16,float16,0,0.4099440097808838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,1,128,1,fp8,fp8,0,0.36974880695343015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,2,128,1,float16,float16,0,0.389900803565979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,2,128,1,float16,fp8,0,0.3786832094192505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,2,128,1,fp8,fp8,0,0.37785279750823975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,4,128,1,float16,fp8,0,0.39681599140167234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,4,128,1,fp8,fp8,0,0.39687039852142336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,8,128,1,float16,float16,0,0.4500751972198486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,8,128,1,float16,fp8,0,0.43276638984680177
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,8,128,1,fp8,fp8,0,0.43161759376525877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,24,128,1,float16,float16,0,0.31981759071350097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,24,128,1,float16,fp8,0,0.30187039375305175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,24,128,1,fp8,fp8,0,0.30071520805358887
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,1,128,1,float16,float16,0,0.2014591932296753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,1,128,1,float16,fp8,0,0.19337600469589233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,1,128,1,fp8,fp8,0,0.19428800344467162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,2,128,1,float16,float16,0,0.21112639904022218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,2,128,1,float16,fp8,0,0.20479679107666016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,2,128,1,fp8,fp8,0,0.20356640815734864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,4,128,1,float16,float16,0,0.21912000179290772
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,4,128,1,float16,fp8,0,0.21167519092559814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,4,128,1,fp8,fp8,0,0.20985119342803954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,8,128,1,float16,float16,0,0.23881919384002687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,8,128,1,float16,fp8,0,0.22893118858337402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,8,128,1,fp8,fp8,0,0.2286223888397217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,1,128,1,float16,float16,0,3.4317264556884766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,1,128,1,float16,fp8,0,3.2387920379638673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,1,128,1,fp8,fp8,0,3.2780799865722656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,2,128,1,float16,float16,0,3.6412384033203127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,2,128,1,float16,fp8,0,3.3661968231201174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,2,128,1,fp8,fp8,0,3.340603256225586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,4,128,1,float16,float16,0,3.9843521118164062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,4,128,1,fp8,fp8,0,3.5547870635986327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,4,128,1,float16,fp8,0,3.6873600006103517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,24,128,1,float16,float16,0,3.0722223281860352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,24,128,1,float16,fp8,0,2.989792060852051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,24,128,1,fp8,fp8,0,2.9184751510620117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,8,128,1,float16,float16,0,4.425761413574219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,8,128,1,float16,fp8,0,4.103995132446289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,8,128,1,fp8,fp8,0,3.9814544677734376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,1,128,1,float16,float16,0,1.7309440612792968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,1,128,1,float16,fp8,0,1.6160415649414062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,1,128,1,fp8,fp8,0,1.640247917175293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,2,128,1,float16,float16,0,1.8176191329956055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,2,128,1,float16,fp8,0,1.677222442626953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,2,128,1,fp8,fp8,0,1.7190927505493163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,4,128,1,float16,float16,0,1.9030111312866211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,4,128,1,float16,fp8,0,1.7888336181640625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,4,128,1,fp8,fp8,0,1.9725391387939453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,8,128,1,float16,float16,0,2.1324464797973635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,8,128,1,float16,fp8,0,2.0323423385620116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,24,128,1,float16,float16,0,1.545188808441162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,1,128,1,float16,float16,0,0.8628720283508301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,8,128,1,fp8,fp8,0,2.113520050048828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,24,128,1,float16,fp8,0,1.4520031929016113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,1,128,1,float16,fp8,0,0.8468624114990234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,24,128,1,fp8,fp8,0,1.5230287551879882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,1,128,1,fp8,fp8,0,0.8714768409729003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,2,128,1,float16,float16,0,0.8922767639160156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,2,128,1,float16,fp8,0,0.8752367973327637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,2,128,1,fp8,fp8,0,0.8761872291564942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,4,128,1,float16,float16,0,0.9668304443359375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,4,128,1,float16,fp8,0,0.910431957244873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,4,128,1,fp8,fp8,0,0.908135986328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,8,128,1,float16,float16,0,1.0701871871948243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,8,128,1,float16,fp8,0,1.048363208770752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,8,128,1,fp8,fp8,0,1.080947208404541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,24,128,1,float16,float16,0,0.7920512199401856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,24,128,1,float16,fp8,0,0.7542064189910889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,1,128,1,float16,float16,0,0.4445824146270752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,24,128,1,fp8,fp8,0,0.750651216506958
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,1,128,1,float16,fp8,0,0.4236800193786621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,1,128,1,fp8,fp8,0,0.4376512050628662
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,2,128,1,float16,float16,0,0.459884786605835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,2,128,1,float16,fp8,0,0.45563521385192873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,2,128,1,fp8,fp8,0,0.4454048156738281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,8,128,1,float16,float16,0,0.5366335868835449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,4,128,1,float16,float16,0,0.4911664009094238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,4,128,1,float16,fp8,0,0.47684478759765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,4,128,1,fp8,fp8,0,0.47489118576049805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,8,128,1,float16,fp8,0,0.5315167903900146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,8,128,1,fp8,fp8,0,0.530395221710205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,24,128,1,float16,float16,0,0.4168816089630127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,24,128,1,float16,fp8,0,0.38598880767822263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,24,128,1,fp8,fp8,0,0.3872751951217651
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,1,128,1,float16,float16,0,0.23524799346923828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,1,128,1,float16,fp8,0,0.228985595703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,1,128,1,fp8,fp8,0,0.22840640544891358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,2,128,1,float16,float16,0,0.24348480701446534
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,2,128,1,float16,fp8,0,0.23632640838623048
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,2,128,1,fp8,fp8,0,0.23715519905090332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,4,128,1,float16,float16,0,0.25692479610443114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,4,128,1,float16,fp8,0,0.2512207984924316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,4,128,1,fp8,fp8,0,0.25176639556884767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,8,128,1,float16,float16,0,0.287992000579834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,1,128,1,float16,float16,0,0.13138079643249512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,8,128,1,float16,fp8,0,0.27877280712127683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,8,128,1,fp8,fp8,0,0.27878239154815676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,24,128,1,float16,float16,0,0.21953918933868408
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,24,128,1,float16,fp8,0,0.2072688102722168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,24,128,1,fp8,fp8,0,0.2070688009262085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,1,128,1,float16,fp8,0,0.12192319631576538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,1,128,1,fp8,fp8,0,0.12222239971160889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,2,128,1,float16,float16,0,0.13411359786987304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,8,128,1,float16,float16,0,0.15741920471191406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,2,128,1,float16,fp8,0,0.1243615984916687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,2,128,1,fp8,fp8,0,0.124617600440979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,4,128,1,float16,float16,0,0.14156639575958252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,4,128,1,float16,fp8,0,0.1361855983734131
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,4,128,1,fp8,fp8,0,0.13521120548248292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,8,128,1,float16,fp8,0,0.151528000831604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,8,128,1,fp8,fp8,0,0.15128320455551147
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,1,128,1,float16,float16,0,3.4107807159423826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,1,128,1,float16,fp8,0,3.2056209564208986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,1,128,1,fp8,fp8,0,3.2201183319091795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,2,128,1,float16,float16,0,3.4327377319335937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,2,128,1,float16,fp8,0,3.3713584899902345
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,2,128,1,fp8,fp8,0,3.347030258178711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,4,128,1,float16,fp8,0,3.6469520568847655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,4,128,1,float16,float16,0,3.9328113555908204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,4,128,1,fp8,fp8,0,3.629663848876953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,24,128,1,float16,float16,0,3.4713470458984377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,8,128,1,float16,float16,0,4.557271957397461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,8,128,1,float16,fp8,0,4.295552062988281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,8,128,1,fp8,fp8,0,4.22229118347168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,24,128,1,float16,fp8,0,3.293308639526367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,24,128,1,fp8,fp8,0,3.3210079193115236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,1,128,1,float16,float16,0,1.7052288055419922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,1,128,1,float16,fp8,0,1.617283248901367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,1,128,1,fp8,fp8,0,1.6724336624145508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,2,128,1,float16,float16,0,1.7335567474365234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,2,128,1,float16,fp8,0,1.6702239990234375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,2,128,1,fp8,fp8,0,1.679102325439453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,4,128,1,float16,float16,0,1.895675277709961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,4,128,1,float16,fp8,0,1.9614416122436524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,4,128,1,fp8,fp8,0,1.8469295501708984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,8,128,1,float16,float16,0,2.20914249420166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,24,128,1,float16,float16,0,1.7195104598999023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,24,128,1,float16,fp8,0,1.6526287078857422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,1,128,1,float16,float16,0,0.8633824348449707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,8,128,1,float16,fp8,0,2.1154800415039063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,8,128,1,fp8,fp8,0,2.197151947021484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,24,128,1,fp8,fp8,0,1.7018863677978515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,1,128,1,float16,fp8,0,0.8144847869873046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,1,128,1,fp8,fp8,0,0.8419872283935547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,2,128,1,float16,float16,0,0.8790271759033204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,2,128,1,float16,fp8,0,0.8465616226196289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,2,128,1,fp8,fp8,0,0.8740575790405274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,4,128,1,float16,float16,0,0.9627903938293457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,4,128,1,float16,fp8,0,0.9437888145446778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,4,128,1,fp8,fp8,0,0.9446784019470215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,8,128,1,float16,fp8,0,1.1000399589538574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,8,128,1,float16,float16,0,1.1089183807373046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,24,128,1,float16,float16,0,0.8669919967651367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,8,128,1,fp8,fp8,0,1.099232006072998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,24,128,1,float16,fp8,0,0.857033634185791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,1,128,1,float16,float16,0,0.4375135898590088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,24,128,1,fp8,fp8,0,0.8554944038391114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,1,128,1,float16,fp8,0,0.4282527923583984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,1,128,1,fp8,fp8,0,0.4211775779724121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,2,128,1,float16,float16,0,0.45379838943481443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,2,128,1,float16,fp8,0,0.43567519187927245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,2,128,1,fp8,fp8,0,0.4470223903656006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,4,128,1,float16,float16,0,0.499454402923584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,4,128,1,float16,fp8,0,0.47661762237548827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,4,128,1,fp8,fp8,0,0.4849520206451416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,8,128,1,float16,float16,0,0.5719279766082763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,8,128,1,float16,fp8,0,0.5470143795013428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,8,128,1,fp8,fp8,0,0.5535103797912597
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,24,128,1,float16,float16,0,0.4462768077850342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,24,128,1,float16,fp8,0,0.44176321029663085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,24,128,1,fp8,fp8,0,0.4425055980682373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,1,128,1,float16,float16,0,0.22871840000152588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,1,128,1,float16,fp8,0,0.22838399410247803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,1,128,1,fp8,fp8,0,0.22842400074005126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,2,128,1,float16,float16,0,0.23709280490875245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,2,128,1,float16,fp8,0,0.23491840362548827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,2,128,1,fp8,fp8,0,0.23101119995117186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,4,128,1,float16,float16,0,0.26163198947906496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,8,128,1,fp8,fp8,0,0.2857696056365967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,4,128,1,float16,fp8,0,0.256329607963562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,4,128,1,fp8,fp8,0,0.2523711919784546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,8,128,1,float16,float16,0,0.3004623889923096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,8,128,1,float16,fp8,0,0.2920880079269409
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,1,128,1,fp8,fp8,0,0.12204320430755615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,24,128,1,float16,float16,0,0.24193599224090576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,24,128,1,float16,fp8,0,0.23125920295715333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,24,128,1,fp8,fp8,0,0.23344318866729735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,1,128,1,float16,float16,0,0.1256368041038513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,1,128,1,float16,fp8,0,0.12452960014343262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,2,128,1,float16,float16,0,0.1347823977470398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,2,128,1,float16,fp8,0,0.1310256004333496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,2,128,1,fp8,fp8,0,0.1285040020942688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,8,128,1,fp8,fp8,0,0.1583631992340088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,4,128,1,float16,float16,0,0.14209120273590087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,24,128,1,float16,fp8,0,0.13194880485534669
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,4,128,1,float16,fp8,0,0.13955680131912232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,4,128,1,fp8,fp8,0,0.14118880033493042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,8,128,1,float16,float16,0,0.15998239517211915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,1,128,1,fp8,fp8,0,0.06937119960784913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,8,128,1,float16,fp8,0,0.15813920497894288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,24,128,1,float16,float16,0,0.13864320516586304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,24,128,1,fp8,fp8,0,0.13201600313186646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,1,128,1,float16,float16,0,0.07200959920883179
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,1,128,1,float16,fp8,0,0.07004799842834472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,2,128,1,float16,float16,0,0.07362080216407776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,2,128,1,float16,fp8,0,0.07133439779281617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,2,128,1,fp8,fp8,0,0.06994240283966065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,4,128,1,float16,float16,0,0.08189280033111572
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,4,128,1,float16,fp8,0,0.0746944010257721
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,4,128,1,fp8,fp8,0,0.07555999755859374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,8,128,1,float16,float16,0,0.09293280243873596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,8,128,1,float16,fp8,0,0.08374559879302979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,8,128,1,fp8,fp8,0,0.08354719877243041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,1,128,1,float16,float16,0,2.064596748352051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,1,128,1,float16,fp8,0,1.9908464431762696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,1,128,1,fp8,fp8,0,2.0100048065185545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,2,128,1,float16,float16,0,2.1661935806274415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,2,128,1,float16,fp8,0,2.106135940551758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,2,128,1,fp8,fp8,0,2.104425621032715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,4,128,1,float16,float16,0,2.425742340087891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,4,128,1,float16,fp8,0,2.3894784927368162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,4,128,1,fp8,fp8,0,2.326468849182129
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,8,128,1,float16,float16,0,2.924502372741699
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,24,128,1,float16,float16,0,2.376961517333984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,8,128,1,float16,fp8,0,2.773054313659668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,8,128,1,fp8,fp8,0,2.763505554199219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,1,128,1,float16,float16,0,1.0183343887329102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,24,128,1,float16,fp8,0,2.3223503112792967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,1,128,1,float16,fp8,0,1.0416367530822754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,1,128,1,fp8,fp8,0,1.016097640991211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,24,128,1,fp8,fp8,0,2.309364891052246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,2,128,1,float16,float16,0,1.0820416450500487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,2,128,1,float16,fp8,0,1.0597583770751953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,2,128,1,fp8,fp8,0,1.0729503631591797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,4,128,1,float16,float16,0,1.215078353881836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,4,128,1,float16,fp8,0,1.2007328033447267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,4,128,1,fp8,fp8,0,1.1884960174560546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,8,128,1,float16,float16,0,1.4493696212768554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,8,128,1,float16,fp8,0,1.4419407844543457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,24,128,1,float16,float16,0,1.1969167709350585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,8,128,1,fp8,fp8,0,1.3908032417297362
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,24,128,1,float16,fp8,0,1.1723888397216797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,1,128,1,float16,float16,0,0.5301424026489258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,1,128,1,float16,fp8,0,0.523033618927002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,24,128,1,fp8,fp8,0,1.170849609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,1,128,1,fp8,fp8,0,0.5245215892791748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,2,128,1,float16,float16,0,0.557316780090332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,2,128,1,float16,fp8,0,0.5419568061828614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,2,128,1,fp8,fp8,0,0.5553760051727294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,4,128,1,float16,float16,0,0.6168591976165771
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,4,128,1,float16,fp8,0,0.6075776100158692
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,4,128,1,fp8,fp8,0,0.609662389755249
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,8,128,1,float16,fp8,0,0.7180160045623779
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,8,128,1,float16,float16,0,0.7372288227081298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,24,128,1,float16,fp8,0,0.6024943828582764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,8,128,1,fp8,fp8,0,0.7224703788757324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,24,128,1,float16,float16,0,0.6155632019042969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,1,128,1,float16,float16,0,0.2786351919174194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,2,128,1,float16,fp8,0,0.29016640186309817
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,24,128,1,fp8,fp8,0,0.5982704162597656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,1,128,1,float16,fp8,0,0.2740816116333008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,1,128,1,fp8,fp8,0,0.27299840450286866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,2,128,1,float16,float16,0,0.2928368091583252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,2,128,1,fp8,fp8,0,0.2863935947418213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,8,128,1,float16,fp8,0,0.3724128007888794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,4,128,1,float16,float16,0,0.32125439643859866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,4,128,1,float16,fp8,0,0.31672799587249756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,4,128,1,fp8,fp8,0,0.31892640590667726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,8,128,1,float16,float16,0,0.38056800365447996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,8,128,1,fp8,fp8,0,0.37101919651031495
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,24,128,1,float16,float16,0,0.32283840179443357
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,2,128,1,float16,float16,0,0.15819679498672484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,24,128,1,float16,fp8,0,0.31322081089019777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,24,128,1,fp8,fp8,0,0.3147648096084595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,1,128,1,float16,float16,0,0.15059679746627808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,1,128,1,float16,fp8,0,0.15051039457321166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,1,128,1,fp8,fp8,0,0.15016000270843505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,2,128,1,float16,fp8,0,0.15637600421905518
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,2,128,1,fp8,fp8,0,0.15580159425735474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,4,128,1,float16,float16,0,0.17320159673690796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,4,128,1,float16,fp8,0,0.16984319686889648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,4,128,1,fp8,fp8,0,0.17189120054244994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,8,128,1,float16,float16,0,0.20302081108093262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,8,128,1,float16,fp8,0,0.1994223952293396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,8,128,1,fp8,fp8,0,0.19931999444961548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,24,128,1,float16,float16,0,0.17414560317993164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,2,128,1,float16,float16,0,0.08934400081634522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,24,128,1,float16,fp8,0,0.17346240282058717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,24,128,1,fp8,fp8,0,0.1728368043899536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,1,128,1,float16,float16,0,0.08770880103111267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,1,128,1,float16,fp8,0,0.08199679851531982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,1,128,1,fp8,fp8,0,0.08126720190048217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,2,128,1,float16,fp8,0,0.0859935998916626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,2,128,1,fp8,fp8,0,0.08408160209655761
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,4,128,1,float16,float16,0,0.09723359942436219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,4,128,1,float16,fp8,0,0.09451040029525756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,4,128,1,fp8,fp8,0,0.09430400133132935
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,8,128,1,float16,float16,0,0.11182719469070435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,8,128,1,float16,fp8,0,0.10991359949111938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,8,128,1,fp8,fp8,0,0.10979199409484863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,24,128,1,float16,float16,0,0.10153919458389282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,24,128,1,float16,fp8,0,0.0950543999671936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,24,128,1,fp8,fp8,0,0.09455999732017517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,1,128,1,float16,float16,0,0.053217601776123044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,1,128,1,float16,fp8,0,0.05135359764099121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,1,128,1,fp8,fp8,0,0.05095360279083252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,2,128,1,float16,float16,0,0.05259360074996948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,2,128,1,float16,fp8,0,0.051894402503967284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,2,128,1,fp8,fp8,0,0.051806402206420896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,4,128,1,float16,float16,0,0.055032002925872806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,4,128,1,float16,fp8,0,0.05400800108909607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,4,128,1,fp8,fp8,0,0.055088001489639285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,8,128,1,float16,float16,0,0.06362400054931641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,8,128,1,float16,fp8,0,0.06064000129699707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,8,128,1,fp8,fp8,0,0.061185598373413086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,1,128,1,float16,float16,0,2.1379344940185545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,1,128,1,float16,fp8,0,2.1306079864501952
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,1,128,1,fp8,fp8,0,2.114441680908203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,2,128,1,float16,float16,0,2.307756805419922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,2,128,1,float16,fp8,0,2.2632287979125976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,2,128,1,fp8,fp8,0,2.2698080062866213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,4,128,1,float16,float16,0,2.659124755859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,4,128,1,float16,fp8,0,2.5854608535766603
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,4,128,1,fp8,fp8,0,2.584579277038574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,8,128,1,float16,float16,0,3.2476558685302734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,8,128,1,float16,fp8,0,3.1943103790283205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,24,128,1,float16,float16,0,2.8785295486450195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,8,128,1,fp8,fp8,0,3.1652751922607423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,1,128,1,float16,float16,0,1.0872096061706542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,24,128,1,float16,fp8,0,2.7841039657592774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,1,128,1,float16,fp8,0,1.1029647827148437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,1,128,1,fp8,fp8,0,1.068393611907959
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,2,128,1,float16,float16,0,1.1506400108337402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,2,128,1,float16,fp8,0,1.1392255783081056
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,2,128,1,fp8,fp8,0,1.1463567733764648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,24,128,1,fp8,fp8,0,2.744620704650879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,4,128,1,float16,float16,0,1.3286720275878907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,4,128,1,float16,fp8,0,1.3179887771606444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,4,128,1,fp8,fp8,0,1.3079952239990233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,8,128,1,float16,float16,0,1.6234016418457031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,8,128,1,float16,fp8,0,1.6027727127075195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,24,128,1,float16,float16,0,1.4701616287231445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,8,128,1,fp8,fp8,0,1.5857487678527833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,1,128,1,float16,float16,0,0.563811206817627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,1,128,1,float16,fp8,0,0.5538288116455078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,24,128,1,float16,fp8,0,1.3857791900634766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,24,128,1,fp8,fp8,0,1.4198320388793946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,1,128,1,fp8,fp8,0,0.5497104167938233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,2,128,1,float16,float16,0,0.5985568046569825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,2,128,1,float16,fp8,0,0.5912303924560547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,2,128,1,fp8,fp8,0,0.5771344184875489
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,4,128,1,float16,float16,0,0.6660016059875489
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,4,128,1,float16,fp8,0,0.6667632102966309
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,8,128,1,float16,fp8,0,0.8196288108825683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,4,128,1,fp8,fp8,0,0.6566239833831787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,8,128,1,float16,float16,0,0.8425423622131347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,8,128,1,fp8,fp8,0,0.8181424140930176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,24,128,1,float16,float16,0,0.7508399963378907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,1,128,1,float16,float16,0,0.2886960029602051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,24,128,1,float16,fp8,0,0.702561616897583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,1,128,1,float16,fp8,0,0.2897455930709839
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,24,128,1,fp8,fp8,0,0.7108128070831299
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,1,128,1,fp8,fp8,0,0.28568320274353026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,2,128,1,float16,float16,0,0.31610400676727296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,2,128,1,float16,fp8,0,0.3046864032745361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,2,128,1,fp8,fp8,0,0.31078081130981444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,4,128,1,float16,float16,0,0.3460047960281372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,4,128,1,float16,fp8,0,0.34090399742126465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,4,128,1,fp8,fp8,0,0.34375998973846433
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,8,128,1,float16,float16,0,0.4234623908996582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,8,128,1,float16,fp8,0,0.41441121101379397
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,8,128,1,fp8,fp8,0,0.4233391761779785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,24,128,1,float16,float16,0,0.3839328050613403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,24,128,1,float16,fp8,0,0.37695040702819826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,24,128,1,fp8,fp8,0,0.36912639141082765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,1,128,1,float16,float16,0,0.15797760486602783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,1,128,1,float16,fp8,0,0.15821759700775145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,1,128,1,fp8,fp8,0,0.15708320140838622
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,2,128,1,float16,float16,0,0.16599040031433104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,2,128,1,float16,fp8,0,0.16785119771957396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,2,128,1,fp8,fp8,0,0.16472320556640624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,4,128,1,float16,float16,0,0.19163360595703124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,24,128,1,float16,float16,0,0.20970399379730226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,4,128,1,float16,fp8,0,0.18676480054855346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,24,128,1,float16,fp8,0,0.1975200057029724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,4,128,1,fp8,fp8,0,0.18331040143966676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,8,128,1,float16,float16,0,0.22663359642028807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,8,128,1,float16,fp8,0,0.22093119621276855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,8,128,1,fp8,fp8,0,0.22055521011352539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,1,128,1,float16,float16,0,0.09229120016098022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,24,128,1,fp8,fp8,0,0.20247840881347656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,1,128,1,float16,fp8,0,0.09111679792404175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,1,128,1,fp8,fp8,0,0.08911839723587037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,2,128,1,float16,float16,0,0.09882240295410157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,2,128,1,float16,fp8,0,0.09642400145530701
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,2,128,1,fp8,fp8,0,0.09738720059394837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,4,128,1,float16,float16,0,0.10539360046386718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,4,128,1,float16,fp8,0,0.10496000051498414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,4,128,1,fp8,fp8,0,0.10580159425735473
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,8,128,1,float16,float16,0,0.12668479681015016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,8,128,1,float16,fp8,0,0.12308640480041504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,8,128,1,fp8,fp8,0,0.12571359872817994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,24,128,1,float16,float16,0,0.11787199974060059
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,24,128,1,float16,fp8,0,0.11472320556640625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,24,128,1,fp8,fp8,0,0.11209280490875244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,1,128,1,float16,float16,0,0.05126399993896484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,1,128,1,float16,fp8,0,0.05160800218582153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,1,128,1,fp8,fp8,0,0.05207520127296448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,2,128,1,float16,float16,0,0.05256800055503845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,2,128,1,float16,fp8,0,0.05251359939575195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,2,128,1,fp8,fp8,0,0.052014398574829104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,4,128,1,float16,float16,0,0.06109920144081116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,4,128,1,float16,fp8,0,0.057704001665115356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,4,128,1,fp8,fp8,0,0.05813120007514953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,8,128,1,float16,float16,0,0.07279360294342041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,8,128,1,float16,fp8,0,0.06733279824256896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,8,128,1,fp8,fp8,0,0.06696640253067017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,24,128,1,float16,float16,0,0.06743040084838867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,24,128,1,float16,fp8,0,0.0601311981678009
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,24,128,1,fp8,fp8,0,0.060027199983596805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,1,128,1,float16,float16,0,0.034825599193573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,1,128,1,float16,fp8,0,0.03540639877319336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,1,128,1,fp8,fp8,0,0.035452800989151004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,2,128,1,float16,float16,0,0.034948799014091494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,2,128,1,float16,fp8,0,0.036027199029922484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,2,128,1,fp8,fp8,0,0.035769599676132205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,4,128,1,float16,float16,0,0.03578239977359772
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,4,128,1,float16,fp8,0,0.03750559985637665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,4,128,1,fp8,fp8,0,0.03797760009765625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,8,128,1,float16,float16,0,0.04095360040664673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,8,128,1,float16,fp8,0,0.042044800519943235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,8,128,1,fp8,fp8,0,0.042243200540542605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,1,128,1,float16,float16,0,1.5352848052978516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,1,128,1,float16,fp8,0,1.5600319862365724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,1,128,1,fp8,fp8,0,1.5717984199523927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,2,128,1,float16,float16,0,1.7025583267211915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,2,128,1,float16,fp8,0,1.7215936660766602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,2,128,1,fp8,fp8,0,1.71246395111084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,4,128,1,float16,float16,0,2.011862373352051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,4,128,1,float16,fp8,0,2.0135936737060547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,4,128,1,fp8,fp8,0,2.014308738708496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,8,128,1,float16,float16,0,2.641110420227051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,1,128,1,float16,float16,0,0.776470422744751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,8,128,1,float16,fp8,0,2.6180112838745115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,1,128,1,float16,fp8,0,0.7925407886505127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,8,128,1,fp8,fp8,0,2.619864082336426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,24,128,1,float16,float16,0,2.5796384811401367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,24,128,1,float16,fp8,0,2.519607925415039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,1,128,1,fp8,fp8,0,0.7926208019256592
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,2,128,1,float16,float16,0,0.8552495956420898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,2,128,1,float16,fp8,0,0.8637663841247558
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,2,128,1,fp8,fp8,0,0.8621151924133301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,24,128,1,fp8,fp8,0,2.5346479415893555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,4,128,1,float16,float16,0,1.0077584266662598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,4,128,1,float16,fp8,0,1.0173775672912597
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,4,128,1,fp8,fp8,0,1.0170144081115722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,8,128,1,float16,float16,0,1.318782424926758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,1,128,1,float16,float16,0,0.401580810546875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,8,128,1,float16,fp8,0,1.3338432312011719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,1,128,1,float16,fp8,0,0.4093520164489746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,8,128,1,fp8,fp8,0,1.3119248390197753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,24,128,1,float16,float16,0,1.291004753112793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,24,128,1,float16,fp8,0,1.2690863609313965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,1,128,1,fp8,fp8,0,0.4133600234985352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,24,128,1,fp8,fp8,0,1.2576592445373536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,2,128,1,float16,float16,0,0.43892478942871094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,2,128,1,float16,fp8,0,0.4504608154296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,2,128,1,fp8,fp8,0,0.4443056106567383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,4,128,1,float16,float16,0,0.5145103931427002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,4,128,1,float16,fp8,0,0.5178415775299072
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,4,128,1,fp8,fp8,0,0.5211487770080566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,8,128,1,float16,float16,0,0.6811071872711182
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,8,128,1,float16,fp8,0,0.6672768115997314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,8,128,1,fp8,fp8,0,0.6781871795654297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,24,128,1,float16,float16,0,0.6734799861907959
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,1,128,1,float16,float16,0,0.21235361099243164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,24,128,1,float16,fp8,0,0.6431712150573731
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,1,128,1,float16,fp8,0,0.21713120937347413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,24,128,1,fp8,fp8,0,0.6418432235717774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,1,128,1,fp8,fp8,0,0.2189552068710327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,2,128,1,float16,float16,0,0.23153600692749024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,2,128,1,float16,fp8,0,0.23752319812774658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,2,128,1,fp8,fp8,0,0.23543360233306884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,4,128,1,float16,float16,0,0.26883840560913086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,4,128,1,float16,fp8,0,0.27118558883666993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,4,128,1,fp8,fp8,0,0.2717839956283569
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,8,128,1,float16,float16,0,0.3457711935043335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,8,128,1,float16,fp8,0,0.3497776031494141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,8,128,1,fp8,fp8,0,0.3514640092849731
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,24,128,1,float16,float16,0,0.3491472005844116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,24,128,1,float16,fp8,0,0.3355024099349976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,1,128,1,float16,float16,0,0.11917599439620971
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,24,128,1,fp8,fp8,0,0.33895039558410645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,1,128,1,float16,fp8,0,0.12162879705429078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,1,128,1,fp8,fp8,0,0.12124480009078979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,2,128,1,float16,float16,0,0.12933599948883057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,2,128,1,float16,fp8,0,0.13105759620666504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,2,128,1,fp8,fp8,0,0.12979520559310914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,4,128,1,float16,float16,0,0.14785439968109132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,4,128,1,float16,fp8,0,0.148307204246521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,4,128,1,fp8,fp8,0,0.150164794921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,8,128,1,float16,float16,0,0.18513439893722533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,8,128,1,float16,fp8,0,0.18707679510116576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,8,128,1,fp8,fp8,0,0.18662879467010499
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,24,128,1,float16,float16,0,0.1873088002204895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,24,128,1,float16,fp8,0,0.18084319829940795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,2,128,1,float16,fp8,0,0.07769119739532471
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,24,128,1,fp8,fp8,0,0.17970240116119385
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,1,128,1,float16,float16,0,0.07083039879798889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,1,128,1,float16,fp8,0,0.07152159810066223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,1,128,1,fp8,fp8,0,0.07142559885978698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,2,128,1,float16,float16,0,0.07753440141677856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,2,128,1,fp8,fp8,0,0.07706720232963563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,4,128,1,float16,float16,0,0.08497920036315917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,4,128,1,float16,fp8,0,0.08622080087661743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,4,128,1,fp8,fp8,0,0.08575999736785889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,8,128,1,float16,float16,0,0.10475200414657593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,8,128,1,float16,fp8,0,0.10639359951019287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,8,128,1,fp8,fp8,0,0.10543040037155152
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,24,128,1,float16,float16,0,0.10589760541915894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,24,128,1,float16,fp8,0,0.10223040580749512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,24,128,1,fp8,fp8,0,0.10274239778518676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,1,128,1,float16,float16,0,0.04349119961261749
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,1,128,1,float16,fp8,0,0.04196479916572571
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,1,128,1,fp8,fp8,0,0.04254400134086609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,2,128,1,float16,float16,0,0.04421760141849518
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,2,128,1,float16,fp8,0,0.04360159933567047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,2,128,1,fp8,fp8,0,0.04305280148983002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,4,128,1,float16,float16,0,0.051841598749160764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,4,128,1,float16,fp8,0,0.04825280010700226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,4,128,1,fp8,fp8,0,0.048281601071357726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,8,128,1,float16,float16,0,0.0640608012676239
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,8,128,1,float16,fp8,0,0.0574720025062561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,8,128,1,fp8,fp8,0,0.05660799741744995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,24,128,1,float16,float16,0,0.06072319746017456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,24,128,1,float16,fp8,0,0.05707039833068848
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,24,128,1,fp8,fp8,0,0.0567471981048584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,1,128,1,float16,float16,0,0.031656000018119815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,1,128,1,float16,fp8,0,0.03147839903831482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,1,128,1,fp8,fp8,0,0.03121120035648346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,2,128,1,float16,float16,0,0.0318015992641449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,2,128,1,float16,fp8,0,0.03148959875106812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,2,128,1,fp8,fp8,0,0.03173440098762512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,4,128,1,float16,float16,0,0.03284319937229156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,4,128,1,float16,fp8,0,0.03408960103988647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,4,128,1,fp8,fp8,0,0.03415200114250183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,8,128,1,float16,float16,0,0.03696640133857727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,8,128,1,float16,fp8,0,0.03861120045185089
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,8,128,1,fp8,fp8,0,0.038447999954223634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,24,128,1,float16,float16,0,0.039134401082992556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,24,128,1,float16,fp8,0,0.03891839981079102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,24,128,1,fp8,fp8,0,0.038868799805641174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,1,128,1,float16,float16,0,0.028035199642181395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,1,128,1,float16,fp8,0,0.02988159954547882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,1,128,1,fp8,fp8,0,0.029340800642967225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,2,128,1,float16,float16,0,0.028142398595809935
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,2,128,1,float16,fp8,0,0.029304000735282897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,2,128,1,fp8,fp8,0,0.029507198929786684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,4,128,1,float16,float16,0,0.02851040065288544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,4,128,1,float16,fp8,0,0.03004319965839386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,4,128,1,fp8,fp8,0,0.030299198627471925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,8,128,1,float16,float16,0,0.02943359911441803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,8,128,1,float16,fp8,0,0.030731201171875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,1,128,1,float16,float16,0,0.6422368049621582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,8,128,1,fp8,fp8,0,0.030687999725341798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,1,128,1,float16,fp8,0,0.6614960193634033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,1,128,1,fp8,fp8,0,0.6670639991760254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,2,128,1,float16,float16,0,0.7235087871551513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,2,128,1,float16,fp8,0,0.7393919944763183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,2,128,1,fp8,fp8,0,0.7351808071136474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,4,128,1,float16,float16,0,0.8783455848693847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,4,128,1,float16,fp8,0,0.895587158203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,4,128,1,fp8,fp8,0,0.8947183609008789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,8,128,1,float16,float16,0,1.1883392333984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,1,128,1,float16,float16,0,0.3344255924224854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,8,128,1,float16,fp8,0,1.2019488334655761
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,1,128,1,float16,fp8,0,0.3461359977722168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,8,128,1,fp8,fp8,0,1.1945088386535645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,24,128,1,float16,float16,0,1.2488304138183595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,24,128,1,float16,fp8,0,1.1875951766967774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,24,128,1,fp8,fp8,0,1.1888992309570312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,1,128,1,fp8,fp8,0,0.34397120475769044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,2,128,1,float16,float16,0,0.36964800357818606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,2,128,1,float16,fp8,0,0.38314080238342285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,2,128,1,fp8,fp8,0,0.38198559284210204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,4,128,1,float16,float16,0,0.4502096176147461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,4,128,1,float16,fp8,0,0.4560272216796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,4,128,1,fp8,fp8,0,0.4569839954376221
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,8,128,1,float16,float16,0,0.6082863807678223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,8,128,1,float16,fp8,0,0.6066175937652588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,8,128,1,fp8,fp8,0,0.60556640625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,24,128,1,float16,float16,0,0.6318655967712402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,1,128,1,float16,float16,0,0.1787184000015259
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,24,128,1,float16,fp8,0,0.6028096199035644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,1,128,1,float16,fp8,0,0.1844640016555786
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,24,128,1,fp8,fp8,0,0.6019919872283935
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,1,128,1,fp8,fp8,0,0.18571679592132567
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,2,128,1,float16,float16,0,0.19791200160980224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,2,128,1,float16,fp8,0,0.2028831958770752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,2,128,1,fp8,fp8,0,0.2029103994369507
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,4,128,1,float16,float16,0,0.23645761013031005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,4,128,1,float16,fp8,0,0.23991999626159669
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,4,128,1,fp8,fp8,0,0.24037280082702636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,8,128,1,float16,float16,0,0.3132560014724731
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,8,128,1,float16,fp8,0,0.314630389213562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,8,128,1,fp8,fp8,0,0.3147007942199707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,24,128,1,float16,float16,0,0.3289680004119873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,24,128,1,float16,fp8,0,0.3143743991851807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,1,128,1,float16,float16,0,0.10230879783630371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,24,128,1,fp8,fp8,0,0.3135119915008545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,1,128,1,float16,fp8,0,0.10609439611434937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,1,128,1,fp8,fp8,0,0.10580639839172364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,2,128,1,float16,float16,0,0.11047999858856201
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,2,128,1,float16,fp8,0,0.11354880332946778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,2,128,1,fp8,fp8,0,0.11315360069274902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,4,128,1,float16,float16,0,0.1311776041984558
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,4,128,1,float16,fp8,0,0.13236160278320314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,4,128,1,fp8,fp8,0,0.1322991967201233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,8,128,1,float16,float16,0,0.16920479536056518
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,8,128,1,float16,fp8,0,0.17021119594573975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,8,128,1,fp8,fp8,0,0.16914880275726318
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,24,128,1,float16,float16,0,0.17768319845199584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,24,128,1,float16,fp8,0,0.1686944007873535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,24,128,1,fp8,fp8,0,0.16774879693984984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,1,128,1,float16,float16,0,0.06266720294952392
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,1,128,1,float16,fp8,0,0.06314079761505127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,1,128,1,fp8,fp8,0,0.06351360082626342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,2,128,1,float16,float16,0,0.06894400119781494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,2,128,1,float16,fp8,0,0.06929600238800049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,2,128,1,fp8,fp8,0,0.06984800100326538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,4,128,1,float16,float16,0,0.07695840001106262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,4,128,1,float16,fp8,0,0.07807679772377014
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,4,128,1,fp8,fp8,0,0.07773119807243348
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,8,128,1,float16,float16,0,0.09668959975242615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,8,128,1,float16,fp8,0,0.09694560170173645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,8,128,1,fp8,fp8,0,0.09704639911651611
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,24,128,1,float16,float16,0,0.10041120052337646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,24,128,1,float16,fp8,0,0.0948415994644165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,24,128,1,fp8,fp8,0,0.09572479724884034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,1,128,1,float16,float16,0,0.03805600106716156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,1,128,1,float16,fp8,0,0.03854719996452331
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,1,128,1,fp8,fp8,0,0.03845759928226471
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,2,128,1,float16,float16,0,0.04030880033969879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,2,128,1,float16,fp8,0,0.039587199687957764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,2,128,1,fp8,fp8,0,0.03901920020580292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,4,128,1,float16,float16,0,0.046051201224327085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,8,128,1,fp8,fp8,0,0.054390400648117065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,4,128,1,float16,fp8,0,0.04408960044384003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,4,128,1,fp8,fp8,0,0.044860801100730895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,8,128,1,float16,float16,0,0.05879999995231629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,8,128,1,float16,fp8,0,0.05405279994010925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,24,128,1,float16,float16,0,0.05741440057754517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,24,128,1,float16,fp8,0,0.05224000215530396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,24,128,1,fp8,fp8,0,0.05143839716911316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,1,128,1,float16,float16,0,0.02953439950942993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,1,128,1,float16,fp8,0,0.03089120090007782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,1,128,1,fp8,fp8,0,0.03091999888420105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,2,128,1,float16,float16,0,0.02980160117149353
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,2,128,1,float16,fp8,0,0.03109920024871826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,2,128,1,fp8,fp8,0,0.03120799958705902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,4,128,1,float16,float16,0,0.030604800581932066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,4,128,1,float16,fp8,0,0.03246560096740723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,4,128,1,fp8,fp8,0,0.032179200649261476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,8,128,1,float16,float16,0,0.035755199193954465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,8,128,1,float16,fp8,0,0.03651039898395538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,24,128,1,float16,float16,0,0.03446080088615418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,8,128,1,fp8,fp8,0,0.03673279881477356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,24,128,1,float16,fp8,0,0.035395199060440065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,2,128,1,float16,fp8,0,0.025817599892616273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,24,128,1,fp8,fp8,0,0.035195198655128476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,1,128,1,float16,float16,0,0.025200000405311583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,1,128,1,float16,fp8,0,0.02603679895401001
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,1,128,1,fp8,fp8,0,0.02606239914894104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,2,128,1,float16,float16,0,0.025099200010299683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,2,128,1,fp8,fp8,0,0.025905600190162657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,4,128,1,float16,float16,0,0.025519999861717223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,4,128,1,float16,fp8,0,0.02610880136489868
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,4,128,1,fp8,fp8,0,0.026267200708389282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,8,128,1,float16,float16,0,0.026305601000785828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,8,128,1,float16,fp8,0,0.026982399821281432
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,8,128,1,fp8,fp8,0,0.026867198944091796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,24,128,1,float16,float16,0,0.027419200539588927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,24,128,1,float16,fp8,0,0.028040000796318056
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,24,128,1,fp8,fp8,0,0.02815679907798767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,1,128,1,float16,float16,0,0.023468799889087677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,1,128,1,float16,fp8,0,0.024638399481773376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,1,128,1,fp8,fp8,0,0.02460319995880127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,8,128,1,float16,float16,0,0.024251200258731842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,2,128,1,float16,float16,0,0.02364960014820099
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,2,128,1,float16,fp8,0,0.02460159957408905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,2,128,1,fp8,fp8,0,0.0246288001537323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,4,128,1,float16,float16,0,0.023894399404525757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,4,128,1,float16,fp8,0,0.025068798661231996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,1,128,1,float16,fp8,0,0.34294240474700927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,4,128,1,fp8,fp8,0,0.024545599520206452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,8,128,1,float16,fp8,0,0.024817599356174468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,8,128,1,fp8,fp8,0,0.024958400428295134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,1,128,1,float16,float16,0,0.3237663984298706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,1,128,1,fp8,fp8,0,0.33992800712585447
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,2,128,1,float16,float16,0,0.35962400436401365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,2,128,1,float16,fp8,0,0.3841792106628418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,2,128,1,fp8,fp8,0,0.38546080589294435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,4,128,1,float16,float16,0,0.43439202308654784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,4,128,1,float16,fp8,0,0.46151041984558105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,4,128,1,fp8,fp8,0,0.4623295783996582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,8,128,1,float16,float16,0,0.5928575992584229
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,8,128,1,float16,fp8,0,0.6160848140716553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,8,128,1,fp8,fp8,0,0.618779182434082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,24,128,1,float16,float16,0,0.6364783763885498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,1,128,1,float16,float16,0,0.17254240512847902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,24,128,1,float16,fp8,0,0.6164976119995117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,1,128,1,float16,fp8,0,0.18216160535812378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,24,128,1,fp8,fp8,0,0.6149151802062989
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,1,128,1,fp8,fp8,0,0.1839359998703003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,2,128,1,float16,float16,0,0.19104479551315307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,2,128,1,float16,fp8,0,0.20545918941497804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,2,128,1,fp8,fp8,0,0.20145120620727539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,4,128,1,float16,float16,0,0.22905759811401366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,8,128,1,float16,fp8,0,0.3198623895645142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,8,128,1,fp8,fp8,0,0.31942880153656006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,4,128,1,float16,fp8,0,0.24389278888702393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,4,128,1,fp8,fp8,0,0.2436079978942871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,8,128,1,float16,float16,0,0.3083904027938843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,24,128,1,float16,float16,0,0.33076159954071044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,24,128,1,float16,fp8,0,0.3175839900970459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,1,128,1,float16,float16,0,0.09832000136375427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,2,128,1,float16,fp8,0,0.11290559768676758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,24,128,1,fp8,fp8,0,0.3174544095993042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,1,128,1,float16,fp8,0,0.10454239845275878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,1,128,1,fp8,fp8,0,0.10464320182800294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,2,128,1,float16,float16,0,0.10686719417572021
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,2,128,1,fp8,fp8,0,0.11333279609680176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,4,128,1,float16,float16,0,0.1257599949836731
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,4,128,1,float16,fp8,0,0.13173439502716064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,4,128,1,fp8,fp8,0,0.13169280290603638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,8,128,1,float16,float16,0,0.16745120286941528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,8,128,1,float16,fp8,0,0.16936639547348023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,8,128,1,fp8,fp8,0,0.16959199905395508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,24,128,1,float16,float16,0,0.17859679460525513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,24,128,1,float16,fp8,0,0.16662399768829345
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,24,128,1,fp8,fp8,0,0.1670207977294922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,1,128,1,float16,float16,0,0.0608240008354187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,1,128,1,float16,fp8,0,0.059969598054885866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,1,128,1,fp8,fp8,0,0.060524797439575194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,2,128,1,float16,float16,0,0.06685119867324829
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,2,128,1,float16,fp8,0,0.06642240285873413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,2,128,1,fp8,fp8,0,0.0666703999042511
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,4,128,1,float16,float16,0,0.0749679982662201
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,4,128,1,float16,fp8,0,0.07534719705581665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,4,128,1,fp8,fp8,0,0.07558240294456482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,8,128,1,float16,float16,0,0.09499520063400269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,8,128,1,float16,fp8,0,0.09437119960784912
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,8,128,1,fp8,fp8,0,0.0943343997001648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,24,128,1,float16,float16,0,0.09790080189704894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,24,128,1,float16,fp8,0,0.09164000153541565
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,24,128,1,fp8,fp8,0,0.09125440120697022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,4,128,1,float16,float16,0,0.04216800034046173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,1,128,1,float16,float16,0,0.0346096009016037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,1,128,1,float16,fp8,0,0.03514080047607422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,1,128,1,fp8,fp8,0,0.03555519878864288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,2,128,1,float16,float16,0,0.03563359975814819
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,2,128,1,float16,fp8,0,0.03624480068683624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,2,128,1,fp8,fp8,0,0.0361407995223999
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,4,128,1,float16,fp8,0,0.040775999426841736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,4,128,1,fp8,fp8,0,0.04127680063247681
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,8,128,1,float16,float16,0,0.05465919971466064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,8,128,1,float16,fp8,0,0.04811680018901825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,8,128,1,fp8,fp8,0,0.04865919947624207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,2,128,1,float16,float16,0,0.028675198554992676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,24,128,1,float16,float16,0,0.056120002269744874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,2,128,1,float16,fp8,0,0.029607999324798583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,24,128,1,float16,fp8,0,0.050172799825668336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,24,128,1,fp8,fp8,0,0.04954079985618591
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,1,128,1,float16,float16,0,0.02836320102214813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,1,128,1,float16,fp8,0,0.02942880094051361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,1,128,1,fp8,fp8,0,0.029607999324798583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,2,128,1,fp8,fp8,0,0.029585599899291992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,4,128,1,float16,float16,0,0.029543998837471008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,4,128,1,float16,fp8,0,0.03038400113582611
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,4,128,1,fp8,fp8,0,0.030638399720191955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,1,128,1,float16,float16,0,0.024233600497245787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,8,128,1,float16,float16,0,0.033980798721313474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,8,128,1,float16,fp8,0,0.034835198521614076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,8,128,1,fp8,fp8,0,0.03495039939880371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,24,128,1,float16,float16,0,0.03297599852085113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,24,128,1,float16,fp8,0,0.03312639892101288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,24,128,1,fp8,fp8,0,0.03341760039329529
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,1,128,1,float16,fp8,0,0.024519999325275422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,1,128,1,fp8,fp8,0,0.02444320023059845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,2,128,1,float16,float16,0,0.023470400273799895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,2,128,1,float16,fp8,0,0.024424000084400176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,2,128,1,fp8,fp8,0,0.02433920055627823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,4,128,1,float16,float16,0,0.024063999950885772
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,4,128,1,float16,fp8,0,0.02455199956893921
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,4,128,1,fp8,fp8,0,0.024294400215148927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,8,128,1,float16,float16,0,0.024915200471878052
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,8,128,1,float16,fp8,0,0.025361600518226623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,8,128,1,fp8,fp8,0,0.025385600328445435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,24,128,1,float16,float16,0,0.025604799389839172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,24,128,1,float16,fp8,0,0.026398399472236635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,24,128,1,fp8,fp8,0,0.02619520127773285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,1,128,1,float16,float16,0,0.022281600534915923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,1,128,1,float16,fp8,0,0.022832000255584718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,1,128,1,fp8,fp8,0,0.023000000417232512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,2,128,1,float16,float16,0,0.022463999688625336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,2,128,1,float16,fp8,0,0.02276960015296936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,2,128,1,fp8,fp8,0,0.02292799949645996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,4,128,1,float16,float16,0,0.02252320051193237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,4,128,1,float16,fp8,0,0.02314399927854538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,4,128,1,fp8,fp8,0,0.023056000471115112
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,8,128,1,float16,float16,0,0.02285439968109131
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,8,128,1,float16,fp8,0,0.02327200025320053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,8,128,1,fp8,fp8,0,0.023270399868488313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,24,128,1,float16,float16,0,0.022191999852657317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,24,128,1,float16,fp8,0,0.022812800109386445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,24,128,1,fp8,fp8,0,0.022886399924755097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,1,128,1,float16,float16,0,0.021052800118923187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,1,128,1,float16,fp8,0,0.021803200244903564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,1,128,1,fp8,fp8,0,0.022040000557899474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,2,128,1,float16,float16,0,0.02138720005750656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,2,128,1,float16,fp8,0,0.021939200162887574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,2,128,1,fp8,fp8,0,0.02200160026550293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,4,128,1,float16,float16,0,0.021393600106239318
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,4,128,1,float16,fp8,0,0.022233599424362184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,4,128,1,fp8,fp8,0,0.02205120027065277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,8,128,1,float16,float16,0,0.021697600185871125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,8,128,1,float16,fp8,0,0.02236959934234619
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,8,128,1,fp8,fp8,0,0.02205280065536499
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,1,128,1,float16,float16,0,0.17254719734191895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,1,128,1,float16,fp8,0,0.18356640338897706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,1,128,1,fp8,fp8,0,0.18271520137786865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,2,128,1,float16,float16,0,0.19046560525894166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,2,128,1,float16,fp8,0,0.20561599731445312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,2,128,1,fp8,fp8,0,0.2071903944015503
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,4,128,1,float16,float16,0,0.23192958831787108
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,8,128,1,float16,float16,0,0.33737759590148925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,4,128,1,float16,fp8,0,0.24401280879974366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,4,128,1,fp8,fp8,0,0.24368479251861572
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,8,128,1,float16,fp8,0,0.3468735933303833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,8,128,1,fp8,fp8,0,0.3463248014450073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,24,128,1,float16,float16,0,0.36487998962402346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,24,128,1,float16,fp8,0,0.3708240032196045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,1,128,1,fp8,fp8,0,0.10548640489578247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,1,128,1,float16,float16,0,0.09753119945526123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,24,128,1,fp8,fp8,0,0.3721791982650757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,1,128,1,float16,fp8,0,0.1051360011100769
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,2,128,1,float16,float16,0,0.10692319869995118
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,2,128,1,float16,fp8,0,0.11348639726638794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,2,128,1,fp8,fp8,0,0.11352800130844116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,4,128,1,float16,float16,0,0.12672959566116332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,4,128,1,float16,fp8,0,0.13224159479141234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,4,128,1,fp8,fp8,0,0.1322991967201233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,8,128,1,float16,float16,0,0.179476797580719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,8,128,1,float16,fp8,0,0.18539040088653563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,8,128,1,fp8,fp8,0,0.1857599973678589
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,24,128,1,float16,float16,0,0.196343994140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,24,128,1,float16,fp8,0,0.19414080381393434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,24,128,1,fp8,fp8,0,0.19424320459365846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,1,128,1,float16,float16,0,0.06039040088653565
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,1,128,1,float16,fp8,0,0.0606768012046814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,1,128,1,fp8,fp8,0,0.06039680242538452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,2,128,1,float16,float16,0,0.06727520227432252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,2,128,1,float16,fp8,0,0.06732640266418458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,2,128,1,fp8,fp8,0,0.06692320108413696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,4,128,1,float16,float16,0,0.0750432014465332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,4,128,1,float16,fp8,0,0.07587360143661499
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,4,128,1,fp8,fp8,0,0.07589920163154602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,8,128,1,float16,float16,0,0.10202720165252685
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,8,128,1,float16,fp8,0,0.10271999835968018
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,8,128,1,fp8,fp8,0,0.10269279479980468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,24,128,1,float16,float16,0,0.10886880159378051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,24,128,1,float16,fp8,0,0.10420639514923095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,24,128,1,fp8,fp8,0,0.1041856050491333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,1,128,1,float16,float16,0,0.03426879942417145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,1,128,1,float16,fp8,0,0.035364800691604616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,1,128,1,fp8,fp8,0,0.035499200224876404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,2,128,1,float16,float16,0,0.035051199793815616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,2,128,1,float16,fp8,0,0.036771199107170104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,2,128,1,fp8,fp8,0,0.03633599877357483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,4,128,1,float16,float16,0,0.04232960045337677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,4,128,1,float16,fp8,0,0.04098080098628998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,4,128,1,fp8,fp8,0,0.040750399231910706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,8,128,1,float16,float16,0,0.05846080183982849
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,8,128,1,float16,fp8,0,0.05372959971427917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,8,128,1,fp8,fp8,0,0.05310879945755005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,24,128,1,float16,float16,0,0.0584384024143219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,24,128,1,float16,fp8,0,0.053339201211929324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,24,128,1,fp8,fp8,0,0.05388000011444092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,1,128,1,float16,float16,0,0.028172799944877626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,1,128,1,float16,fp8,0,0.02948479950428009
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,1,128,1,fp8,fp8,0,0.029558399319648744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,2,128,1,float16,float16,0,0.028567999601364136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,2,128,1,float16,fp8,0,0.029846400022506714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,2,128,1,fp8,fp8,0,0.029583999514579774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,4,128,1,float16,float16,0,0.02925119996070862
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,4,128,1,float16,fp8,0,0.030399999022483824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,4,128,1,fp8,fp8,0,0.03049919903278351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,8,128,1,float16,float16,0,0.03323520123958588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,8,128,1,float16,fp8,0,0.03459360003471375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,8,128,1,fp8,fp8,0,0.03450559973716736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,24,128,1,float16,float16,0,0.035897600650787356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,24,128,1,float16,fp8,0,0.037487998604774475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,24,128,1,fp8,fp8,0,0.03743039965629578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,1,128,1,float16,float16,0,0.023614400625228883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,1,128,1,float16,fp8,0,0.02427999973297119
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,1,128,1,fp8,fp8,0,0.024099199473857878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,2,128,1,float16,float16,0,0.023316800594329834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,2,128,1,float16,fp8,0,0.02404959946870804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,2,128,1,fp8,fp8,0,0.024060800671577454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,4,128,1,float16,float16,0,0.023982399702072145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,4,128,1,float16,fp8,0,0.024452799558639528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,4,128,1,fp8,fp8,0,0.024475200474262236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,8,128,1,float16,float16,0,0.024435199797153473
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,8,128,1,float16,fp8,0,0.025076800584793092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,8,128,1,fp8,fp8,0,0.025204798579216002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,24,128,1,float16,float16,0,0.02489279955625534
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,24,128,1,float16,fp8,0,0.02566080093383789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,24,128,1,fp8,fp8,0,0.025772801041603087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,1,128,1,float16,float16,0,0.02213120013475418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,1,128,1,float16,fp8,0,0.022489599883556366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,1,128,1,fp8,fp8,0,0.02287999987602234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,2,128,1,float16,float16,0,0.022412799298763275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,2,128,1,float16,fp8,0,0.022945599257946016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,2,128,1,fp8,fp8,0,0.02304159998893738
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,4,128,1,float16,float16,0,0.022356800734996796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,4,128,1,float16,fp8,0,0.02311519980430603
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,4,128,1,fp8,fp8,0,0.023095999658107758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,8,128,1,float16,float16,0,0.02218720018863678
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,8,128,1,float16,fp8,0,0.02359520047903061
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,8,128,1,fp8,fp8,0,0.023012800514698027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,24,128,1,float16,float16,0,0.021862399578094483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,24,128,1,float16,fp8,0,0.02255840003490448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,24,128,1,fp8,fp8,0,0.022433599829673766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,1,128,1,float16,float16,0,0.02106879949569702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,1,128,1,float16,fp8,0,0.021924799680709837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,1,128,1,fp8,fp8,0,0.022006399929523468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,2,128,1,float16,float16,0,0.021139200031757354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,2,128,1,float16,fp8,0,0.02221280038356781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,2,128,1,fp8,fp8,0,0.022329600155353548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,4,128,1,float16,float16,0,0.021358400583267212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,4,128,1,float16,fp8,0,0.02218399941921234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,4,128,1,fp8,fp8,0,0.02207999974489212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,8,128,1,float16,float16,0,0.02136320024728775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,8,128,1,float16,fp8,0,0.02259040027856827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,8,128,1,fp8,fp8,0,0.022100800275802614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,24,128,1,float16,float16,0,0.020947200059890748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,24,128,1,float16,fp8,0,0.02191520035266876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,24,128,1,fp8,fp8,0,0.021731199324131013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,1,128,1,float16,float16,0,0.020508800446987153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,1,128,1,float16,fp8,0,0.021353599429130555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,1,128,1,fp8,fp8,0,0.021377600729465485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,2,128,1,float16,float16,0,0.020715199410915375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,2,128,1,float16,fp8,0,0.021721599996089934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,2,128,1,fp8,fp8,0,0.021457600593566894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,4,128,1,float16,float16,0,0.0208079993724823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,4,128,1,float16,fp8,0,0.02138399928808212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,4,128,1,fp8,fp8,0,0.021484799683094025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,8,128,1,float16,float16,0,0.02109919935464859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,1,128,1,fp8,fp8,0,0.10642720460891723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,8,128,1,float16,fp8,0,0.02163040041923523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,8,128,1,fp8,fp8,0,0.02171040028333664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,1,128,1,float16,float16,0,0.09885759949684143
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,1,128,1,float16,fp8,0,0.10610239505767823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,2,128,1,float16,float16,0,0.1076367974281311
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,2,128,1,float16,fp8,0,0.11461600065231323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,2,128,1,fp8,fp8,0,0.11462080478668213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,4,128,1,float16,float16,0,0.14108959436416627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,4,128,1,float16,fp8,0,0.1482383966445923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,4,128,1,fp8,fp8,0,0.14858399629592894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,8,128,1,float16,float16,0,0.18000320196151734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,8,128,1,float16,fp8,0,0.18685760498046874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,8,128,1,fp8,fp8,0,0.18677279949188233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,24,128,1,float16,float16,0,0.2377903938293457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,24,128,1,float16,fp8,0,0.2477407932281494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,1,128,1,float16,float16,0,0.060547202825546265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,24,128,1,fp8,fp8,0,0.24726400375366211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,1,128,1,float16,fp8,0,0.06178560256958008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,1,128,1,fp8,fp8,0,0.061689597368240354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,2,128,1,float16,float16,0,0.0669647991657257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,2,128,1,float16,fp8,0,0.06873440146446227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,2,128,1,fp8,fp8,0,0.06904640197753906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,4,128,1,float16,float16,0,0.08269439935684204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,4,128,1,float16,fp8,0,0.08460000157356262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,4,128,1,fp8,fp8,0,0.08478720188140869
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,8,128,1,float16,float16,0,0.10271519422531128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,8,128,1,float16,fp8,0,0.10390559434890748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,8,128,1,fp8,fp8,0,0.10375200510025025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,24,128,1,float16,float16,0,0.13006080389022828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,24,128,1,float16,fp8,0,0.1331007957458496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,24,128,1,fp8,fp8,0,0.1324112057685852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,1,128,1,float16,float16,0,0.0341839998960495
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,1,128,1,float16,fp8,0,0.035831999778747556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,1,128,1,fp8,fp8,0,0.03606080114841461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,2,128,1,float16,float16,0,0.03577120006084442
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,2,128,1,float16,fp8,0,0.036734399199485776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,2,128,1,fp8,fp8,0,0.0366127997636795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,4,128,1,float16,float16,0,0.04705919921398163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,4,128,1,float16,fp8,0,0.045070400834083556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,4,128,1,fp8,fp8,0,0.04621120095252991
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,8,128,1,float16,float16,0,0.05986080169677734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,8,128,1,float16,fp8,0,0.053620797395706174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,8,128,1,fp8,fp8,0,0.05472000241279602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,24,128,1,float16,float16,0,0.07072960138320923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,24,128,1,float16,fp8,0,0.06761919856071472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,24,128,1,fp8,fp8,0,0.06738399863243102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,1,128,1,float16,float16,0,0.028409600257873535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,1,128,1,float16,fp8,0,0.029899200797080992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,1,128,1,fp8,fp8,0,0.02967360019683838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,2,128,1,float16,float16,0,0.02858720123767853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,2,128,1,float16,fp8,0,0.029849600791931153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,2,128,1,fp8,fp8,0,0.030112001299858093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,4,128,1,float16,float16,0,0.02924799919128418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,4,128,1,float16,fp8,0,0.030747199058532716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,4,128,1,fp8,fp8,0,0.030687999725341798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,8,128,1,float16,float16,0,0.033718401193618776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,8,128,1,float16,fp8,0,0.03459199965000152
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,8,128,1,fp8,fp8,0,0.03493280112743378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,24,128,1,float16,float16,0,0.03994239866733551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,24,128,1,float16,fp8,0,0.04164640009403229
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,24,128,1,fp8,fp8,0,0.041886401176452634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,1,128,1,float16,float16,0,0.023588800430297853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,1,128,1,float16,fp8,0,0.02423039972782135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,1,128,1,fp8,fp8,0,0.024208000302314757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,2,128,1,float16,float16,0,0.023713600635528565
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,2,128,1,float16,fp8,0,0.024539199471473695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,2,128,1,fp8,fp8,0,0.023979200422763823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,4,128,1,float16,float16,0,0.023558400571346283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,4,128,1,float16,fp8,0,0.024240000545978545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,4,128,1,fp8,fp8,0,0.024383999407291412
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,8,128,1,float16,float16,0,0.024296000599861145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,1,128,1,float16,float16,0,0.022144000232219695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,8,128,1,float16,fp8,0,0.02526719868183136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,8,128,1,fp8,fp8,0,0.025204798579216002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,24,128,1,float16,float16,0,0.02873600125312805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,24,128,1,float16,fp8,0,0.03009440004825592
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,24,128,1,fp8,fp8,0,0.030136001110076905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,1,128,1,float16,fp8,0,0.02279839962720871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,1,128,1,fp8,fp8,0,0.022840000689029694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,2,128,1,float16,float16,0,0.022316800057888032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,2,128,1,float16,fp8,0,0.023048000037670137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,2,128,1,fp8,fp8,0,0.02279199957847595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,4,128,1,float16,float16,0,0.022092799842357635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,4,128,1,float16,fp8,0,0.02292319983243942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,4,128,1,fp8,fp8,0,0.022884799540042876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,8,128,1,float16,float16,0,0.022230400145053862
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,8,128,1,float16,fp8,0,0.023001599311828613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,8,128,1,fp8,fp8,0,0.023003199696540834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,24,128,1,float16,float16,0,0.02173600047826767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,24,128,1,float16,fp8,0,0.022697600722312927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,24,128,1,fp8,fp8,0,0.02266560047864914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,1,128,1,float16,float16,0,0.02128800004720688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,1,128,1,float16,fp8,0,0.021990400552749634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,1,128,1,fp8,fp8,0,0.021996800601482392
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,2,128,1,float16,float16,0,0.02109439969062805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,2,128,1,float16,fp8,0,0.022097599506378175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,2,128,1,fp8,fp8,0,0.022011199593544008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,4,128,1,float16,float16,0,0.02160799950361252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,4,128,1,float16,fp8,0,0.02253919988870621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,4,128,1,fp8,fp8,0,0.022368000447750093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,8,128,1,float16,float16,0,0.021091200411319733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,8,128,1,float16,fp8,0,0.021886399388313292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,8,128,1,fp8,fp8,0,0.022324800491333008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,24,128,1,float16,float16,0,0.020897600054740905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,24,128,1,float16,fp8,0,0.021775999665260316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,24,128,1,fp8,fp8,0,0.021905599534511565
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,1,128,1,float16,float16,0,0.02072480022907257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,1,128,1,float16,fp8,0,0.021342399716377258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,1,128,1,fp8,fp8,0,0.021411199867725373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,2,128,1,float16,float16,0,0.020561599731445314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,2,128,1,float16,fp8,0,0.021692800521850585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,2,128,1,fp8,fp8,0,0.02147520035505295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,4,128,1,float16,float16,0,0.020937600731849672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,24,128,1,float16,fp8,0,0.021513600647449494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,4,128,1,float16,fp8,0,0.021516799926757812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,4,128,1,fp8,fp8,0,0.02152000069618225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,8,128,1,float16,float16,0,0.02067359983921051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,8,128,1,float16,fp8,0,0.02155199944972992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,8,128,1,fp8,fp8,0,0.021488000452518464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,24,128,1,float16,float16,0,0.021038399636745454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,24,128,1,fp8,fp8,0,0.021819199621677398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,1,128,1,float16,float16,0,0.0203247994184494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,1,128,1,float16,fp8,0,0.021254399418830873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,1,128,1,fp8,fp8,0,0.021303999423980712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,2,128,1,float16,float16,0,0.020179200172424316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,2,128,1,float16,fp8,0,0.02131039947271347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,2,128,1,fp8,fp8,0,0.021297599375247955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,4,128,1,float16,float16,0,0.02056799978017807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,4,128,1,float16,fp8,0,0.021353599429130555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,4,128,1,fp8,fp8,0,0.021140800416469575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,8,128,1,float16,float16,0,0.020483200252056123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,8,128,1,float16,fp8,0,0.021246400475502015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,8,128,1,fp8,fp8,0,0.02107519954442978
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,1,128,1,float16,float16,0,0.0625328004360199
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,1,128,1,float16,fp8,0,0.06360160112380982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,4,128,1,float16,fp8,0,0.08642719984054566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,1,128,1,fp8,fp8,0,0.06238880157470703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,2,128,1,float16,float16,0,0.07559679746627808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,2,128,1,float16,fp8,0,0.07785120010375976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,8,128,1,fp8,fp8,0,0.14049760103225709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,2,128,1,fp8,fp8,0,0.07712159752845764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,24,128,1,float16,float16,0,0.17549279928207398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,4,128,1,float16,float16,0,0.08463199734687805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,4,128,1,fp8,fp8,0,0.08657119870185852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,1,128,1,fp8,fp8,0,0.036881598830223086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,8,128,1,float16,float16,0,0.13082560300827026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,8,128,1,float16,fp8,0,0.14008159637451173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,24,128,1,float16,fp8,0,0.18935359716415406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,24,128,1,fp8,fp8,0,0.18936799764633178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,1,128,1,float16,float16,0,0.035436800122261046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,1,128,1,float16,fp8,0,0.03692319989204407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,4,128,1,fp8,fp8,0,0.04673120081424713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,2,128,1,float16,float16,0,0.03987680077552795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,2,128,1,float16,fp8,0,0.041889598965644835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,2,128,1,fp8,fp8,0,0.04217439889907837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,4,128,1,float16,float16,0,0.04713920056819916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,4,128,1,float16,fp8,0,0.04654400050640106
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,8,128,1,float16,float16,0,0.07560160160064697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,8,128,1,float16,fp8,0,0.07157599925994873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,8,128,1,fp8,fp8,0,0.07180799841880799
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,24,128,1,float16,float16,0,0.09361439943313599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,24,128,1,float16,fp8,0,0.09535199999809266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,24,128,1,fp8,fp8,0,0.0947920024394989
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,1,128,1,float16,float16,0,0.028833600878715514
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,1,128,1,float16,fp8,0,0.03055039942264557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,1,128,1,fp8,fp8,0,0.030665600299835206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,2,128,1,float16,float16,0,0.028673601150512696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,2,128,1,float16,fp8,0,0.030508801341056824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,2,128,1,fp8,fp8,0,0.03071039915084839
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,4,128,1,float16,float16,0,0.02951039969921112
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,4,128,1,float16,fp8,0,0.031012800335884095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,4,128,1,fp8,fp8,0,0.031196799874305726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,8,128,1,float16,float16,0,0.041171199083328246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,8,128,1,float16,fp8,0,0.04413279891014099
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,8,128,1,fp8,fp8,0,0.043966400623321536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,24,128,1,float16,float16,0,0.051044797897338866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,24,128,1,float16,fp8,0,0.05550720095634461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,24,128,1,fp8,fp8,0,0.056006401777267456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,2,128,1,fp8,fp8,0,0.024414399266242982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,1,128,1,float16,float16,0,0.023827199637889863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,1,128,1,float16,fp8,0,0.02446399927139282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,1,128,1,fp8,fp8,0,0.024795199930667877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,2,128,1,float16,float16,0,0.023636800050735474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,2,128,1,float16,fp8,0,0.024609600007534028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,4,128,1,float16,float16,0,0.02417919933795929
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,4,128,1,float16,fp8,0,0.02457759976387024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,4,128,1,fp8,fp8,0,0.024721600115299225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,8,128,1,float16,float16,0,0.028377598524093627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,8,128,1,float16,fp8,0,0.02993920147418976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,8,128,1,fp8,fp8,0,0.02967039942741394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,24,128,1,float16,float16,0,0.03267039954662323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,24,128,1,float16,fp8,0,0.034668800234794614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,24,128,1,fp8,fp8,0,0.03450720012187958
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,1,128,1,float16,float16,0,0.02191520035266876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,1,128,1,float16,fp8,0,0.023014399409294128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,1,128,1,fp8,fp8,0,0.022859199345111846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,2,128,1,float16,float16,0,0.02210240066051483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,2,128,1,float16,fp8,0,0.0229312002658844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,2,128,1,fp8,fp8,0,0.023201599717140198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,4,128,1,float16,float16,0,0.022015999257564544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,4,128,1,float16,fp8,0,0.022776000201702118
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,4,128,1,fp8,fp8,0,0.023043200373649597
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,8,128,1,float16,float16,0,0.02239840030670166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,8,128,1,float16,fp8,0,0.02316479980945587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,8,128,1,fp8,fp8,0,0.02325119972229004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,24,128,1,float16,float16,0,0.025561600923538208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,24,128,1,float16,fp8,0,0.02701599895954132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,24,128,1,fp8,fp8,0,0.026902401447296144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,1,128,1,float16,float16,0,0.021414400637149812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,1,128,1,float16,fp8,0,0.02192640006542206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,1,128,1,fp8,fp8,0,0.021980799734592438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,2,128,1,float16,float16,0,0.021264000236988066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,2,128,1,float16,fp8,0,0.022171199321746826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,2,128,1,fp8,fp8,0,0.021883200109004974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,4,128,1,float16,float16,0,0.021294400095939636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,4,128,1,float16,fp8,0,0.02221280038356781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,4,128,1,fp8,fp8,0,0.02223999947309494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,8,128,1,float16,float16,0,0.02110559940338135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,8,128,1,float16,fp8,0,0.022235199809074402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,8,128,1,fp8,fp8,0,0.021991999447345735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,2,128,1,float16,float16,0,0.02083200067281723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,24,128,1,float16,float16,0,0.02120800018310547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,24,128,1,float16,fp8,0,0.022044800221920013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,24,128,1,fp8,fp8,0,0.022103999555110932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,1,128,1,float16,float16,0,0.02064639925956726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,1,128,1,float16,fp8,0,0.021480000019073485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,1,128,1,fp8,fp8,0,0.02125120013952255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,2,128,1,float16,fp8,0,0.02128159999847412
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,2,128,1,fp8,fp8,0,0.02179519981145859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,4,128,1,float16,float16,0,0.020744000375270844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,4,128,1,float16,fp8,0,0.02139039933681488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,4,128,1,fp8,fp8,0,0.021427200734615327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,8,128,1,float16,float16,0,0.020870399475097657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,8,128,1,float16,fp8,0,0.021753600239753722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,8,128,1,fp8,fp8,0,0.021961599588394165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,24,128,1,float16,float16,0,0.021060800552368163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,24,128,1,float16,fp8,0,0.021715199947357176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,24,128,1,fp8,fp8,0,0.021881599724292756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,4,128,1,float16,float16,0,0.020393599569797517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,1,128,1,float16,float16,0,0.0204927995800972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,4,128,1,float16,fp8,0,0.021484799683094025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,1,128,1,float16,fp8,0,0.021476800739765167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,1,128,1,fp8,fp8,0,0.021147200465202333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,2,128,1,float16,float16,0,0.02056639939546585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,2,128,1,float16,fp8,0,0.021086399257183076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,2,128,1,fp8,fp8,0,0.021252800524234772
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,4,128,1,fp8,fp8,0,0.02117599993944168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,8,128,1,float16,float16,0,0.0204927995800972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,8,128,1,float16,fp8,0,0.021529600024223328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,8,128,1,fp8,fp8,0,0.02136639952659607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,24,128,1,float16,float16,0,0.020766399800777435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,24,128,1,float16,fp8,0,0.02162719964981079
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,24,128,1,fp8,fp8,0,0.02147040069103241
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,1,128,1,float16,float16,0,0.020372800529003143
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,1,128,1,float16,fp8,0,0.021305599808692934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,1,128,1,fp8,fp8,0,0.021372799575328828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,2,128,1,float16,float16,0,0.020550400018692017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,2,128,1,float16,fp8,0,0.02133280038833618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,2,128,1,fp8,fp8,0,0.021216000616550445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,4,128,1,float16,float16,0,0.020478400588035583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,4,128,1,float16,fp8,0,0.021241599321365358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,4,128,1,fp8,fp8,0,0.021372799575328828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,8,128,1,float16,float16,0,0.02040479928255081
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,8,128,1,float16,fp8,0,0.021299199759960176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,8,128,1,fp8,fp8,0,0.021089600026607515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,1,128,1,float16,fp8,0,12.42859344482422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,1,128,1,fp8,fp8,0,12.471937561035157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,2,128,1,float16,fp8,0,12.664166259765626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,2,128,1,fp8,fp8,0,12.661521911621094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,1,128,1,float16,float16,0,15.209169006347656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,2,128,1,float16,float16,0,15.16686553955078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,4,128,1,float16,float16,0,15.331158447265626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,4,128,1,float16,fp8,0,12.866615295410156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,16,128,1,float16,fp8,0,7.322096252441407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,16,128,1,float16,float16,0,8.689647674560547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,16,128,1,fp8,fp8,0,7.288543701171875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,4,128,1,fp8,fp8,0,12.854330444335938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,8,128,1,float16,fp8,0,13.477195739746094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,8,128,1,fp8,fp8,0,13.405158996582031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,1,128,1,float16,float16,0,7.410212707519531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,8,128,1,float16,float16,0,15.77174072265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,1,128,1,float16,fp8,0,6.224943923950195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,1,128,1,fp8,fp8,0,6.349270248413086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,2,128,1,float16,float16,0,7.717704010009766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,2,128,1,float16,fp8,0,6.385115051269532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,2,128,1,fp8,fp8,0,6.393915176391602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,4,128,1,float16,fp8,0,6.4561408996582035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,4,128,1,float16,float16,0,8.387725067138671
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,4,128,1,fp8,fp8,0,7.1524208068847654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,16,128,1,float16,float16,0,4.6911663055419925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,16,128,1,float16,fp8,0,4.173238372802734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,16,128,1,fp8,fp8,0,3.662945556640625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,8,128,1,float16,fp8,0,7.983787536621094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,8,128,1,float16,float16,0,9.347742462158203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,8,128,1,fp8,fp8,0,7.3881980895996096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,1,128,1,float16,fp8,0,3.3948894500732423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,1,128,1,float16,float16,0,4.274065780639648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,1,128,1,fp8,fp8,0,3.681827163696289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,2,128,1,float16,fp8,0,3.1621551513671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,2,128,1,fp8,fp8,0,3.271492767333984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,2,128,1,float16,float16,0,4.208367919921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,4,128,1,float16,fp8,0,3.2994415283203127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,4,128,1,float16,float16,0,3.9381649017333986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,4,128,1,fp8,fp8,0,3.2394382476806642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,16,128,1,float16,float16,0,2.134257507324219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,8,128,1,float16,float16,0,4.136577606201172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,8,128,1,float16,fp8,0,3.3776878356933593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,16,128,1,fp8,fp8,0,1.8596351623535157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,1,128,1,float16,float16,0,1.887481689453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,16,128,1,float16,fp8,0,2.179502487182617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,1,128,1,float16,fp8,0,1.585649585723877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,8,128,1,fp8,fp8,0,3.808736038208008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,1,128,1,fp8,fp8,0,1.759427261352539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,2,128,1,float16,fp8,0,1.5811615943908692
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,2,128,1,float16,float16,0,1.8274864196777343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,2,128,1,fp8,fp8,0,1.589782428741455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,4,128,1,float16,float16,0,1.880526351928711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,4,128,1,float16,fp8,0,1.701576042175293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,4,128,1,fp8,fp8,0,1.6307472229003905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,8,128,1,float16,float16,0,2.000931167602539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,8,128,1,float16,fp8,0,1.7376848220825196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,8,128,1,fp8,fp8,0,1.7053295135498048
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,1,128,1,float16,fp8,0,7.194509124755859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,1,128,1,fp8,fp8,0,7.172447967529297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,2,128,1,float16,fp8,0,7.233529663085937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,1,128,1,float16,float16,0,8.553990173339844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,2,128,1,fp8,fp8,0,7.402513885498047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,2,128,1,float16,float16,0,8.698923492431641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,4,128,1,float16,fp8,0,7.444827270507813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,4,128,1,float16,float16,0,9.14910888671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,16,128,1,float16,fp8,0,4.406472015380859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,16,128,1,fp8,fp8,0,4.365081787109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,16,128,1,float16,float16,0,5.1227375030517575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,4,128,1,fp8,fp8,0,7.464201354980469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,1,128,1,float16,float16,0,4.304609680175782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,8,128,1,float16,fp8,0,7.8572944641113285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,8,128,1,fp8,fp8,0,7.866324615478516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,8,128,1,float16,float16,0,9.375755310058594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,1,128,1,float16,fp8,0,3.601337432861328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,1,128,1,fp8,fp8,0,3.55570068359375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,2,128,1,float16,float16,0,4.228334426879883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,2,128,1,float16,fp8,0,3.630419158935547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,2,128,1,fp8,fp8,0,3.746273422241211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,4,128,1,float16,fp8,0,3.7374481201171874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,4,128,1,float16,float16,0,4.389628982543945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,4,128,1,fp8,fp8,0,3.7969024658203123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,16,128,1,float16,float16,0,2.56133918762207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,16,128,1,float16,fp8,0,2.2179519653320314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,8,128,1,float16,fp8,0,3.9307918548583984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,8,128,1,float16,float16,0,4.6580047607421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,16,128,1,fp8,fp8,0,2.2063888549804687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,8,128,1,fp8,fp8,0,4.1272846221923825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,1,128,1,float16,fp8,0,1.802332878112793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,1,128,1,float16,float16,0,2.3673919677734374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,1,128,1,fp8,fp8,0,2.0352575302124025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,2,128,1,float16,fp8,0,1.8271551132202148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,2,128,1,float16,float16,0,2.178656005859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,2,128,1,fp8,fp8,0,1.9966096878051758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,4,128,1,float16,fp8,0,1.8836944580078125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,4,128,1,float16,float16,0,2.1926624298095705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,4,128,1,fp8,fp8,0,2.1518096923828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,8,128,1,float16,fp8,0,1.997969627380371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,16,128,1,float16,float16,0,1.2780768394470214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,8,128,1,float16,float16,0,2.339244842529297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,16,128,1,float16,fp8,0,1.1318911552429198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,8,128,1,fp8,fp8,0,2.284699249267578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,16,128,1,fp8,fp8,0,1.365777587890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,1,128,1,float16,float16,0,0.9904623985290527
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,1,128,1,float16,fp8,0,0.9435263633728027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,1,128,1,fp8,fp8,0,0.9530816078186035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,2,128,1,float16,fp8,0,0.9474063873291015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,2,128,1,float16,float16,0,1.1100128173828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,2,128,1,fp8,fp8,0,0.9465215682983399
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,4,128,1,float16,float16,0,1.1527520179748536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,4,128,1,float16,fp8,0,1.0849856376647948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,4,128,1,fp8,fp8,0,0.9595487594604493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,8,128,1,float16,float16,0,1.1178223609924316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,8,128,1,float16,fp8,0,1.0304415702819825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,8,128,1,fp8,fp8,0,1.136678409576416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,1,128,1,float16,fp8,0,5.286772918701172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,1,128,1,fp8,fp8,0,5.137807846069336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,1,128,1,float16,float16,0,6.440284729003906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,2,128,1,fp8,fp8,0,5.129875183105469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,2,128,1,float16,fp8,0,5.387635040283203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,2,128,1,float16,float16,0,6.718244934082032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,4,128,1,float16,fp8,0,5.297953414916992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,4,128,1,float16,float16,0,6.530446624755859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,16,128,1,float16,float16,0,3.7424480438232424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,1,128,1,float16,float16,0,2.8749200820922853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,16,128,1,float16,fp8,0,3.414379119873047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,16,128,1,fp8,fp8,0,3.196614456176758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,4,128,1,fp8,fp8,0,5.334985733032227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,8,128,1,fp8,fp8,0,5.715012741088867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,8,128,1,float16,fp8,0,5.7450927734375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,8,128,1,float16,float16,0,6.8188636779785154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,1,128,1,float16,fp8,0,2.6449968338012697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,1,128,1,fp8,fp8,0,2.706345558166504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,2,128,1,float16,fp8,0,2.565731239318848
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,2,128,1,float16,float16,0,2.8656368255615234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,2,128,1,fp8,fp8,0,2.5706880569458006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,4,128,1,float16,fp8,0,2.670427131652832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,4,128,1,float16,float16,0,3.172323226928711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,4,128,1,fp8,fp8,0,2.938091278076172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,16,128,1,float16,fp8,0,1.6370208740234375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,16,128,1,float16,float16,0,1.8958831787109376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,8,128,1,float16,fp8,0,2.879267120361328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,8,128,1,float16,float16,0,3.3258190155029297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,8,128,1,fp8,fp8,0,2.8392528533935546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,16,128,1,fp8,fp8,0,1.6405664443969727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,1,128,1,float16,float16,0,1.5008303642272949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,1,128,1,fp8,fp8,0,1.2783647537231446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,1,128,1,float16,fp8,0,1.309926414489746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,2,128,1,float16,float16,0,1.4424240112304687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,2,128,1,float16,fp8,0,1.3023903846740723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,2,128,1,fp8,fp8,0,1.4138640403747558
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,4,128,1,float16,float16,0,1.4680191993713378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,4,128,1,float16,fp8,0,1.3904047966003419
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,4,128,1,fp8,fp8,0,1.359182357788086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,8,128,1,float16,float16,0,1.5651151657104492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,16,128,1,float16,float16,0,0.9278639793395996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,8,128,1,float16,fp8,0,1.4358832359313964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,8,128,1,fp8,fp8,0,1.4918880462646484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,16,128,1,float16,fp8,0,0.8709952354431152
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,1,128,1,float16,float16,0,0.7094016075134277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,16,128,1,fp8,fp8,0,0.9277183532714843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,1,128,1,float16,fp8,0,0.6562816143035889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,1,128,1,fp8,fp8,0,0.6991504192352295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,2,128,1,float16,float16,0,0.7126560211181641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,2,128,1,float16,fp8,0,0.6681168079376221
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,2,128,1,fp8,fp8,0,0.6800111770629883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,4,128,1,float16,float16,0,0.7424496173858642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,4,128,1,float16,fp8,0,0.7093039989471436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,4,128,1,fp8,fp8,0,0.7140511989593505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,8,128,1,float16,float16,0,0.8002688407897949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,8,128,1,float16,fp8,0,0.7666384220123291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,8,128,1,fp8,fp8,0,0.7449312210083008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,1,128,1,float16,fp8,0,6.5503662109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,1,128,1,fp8,fp8,0,6.6573936462402346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,2,128,1,float16,fp8,0,6.695587158203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,1,128,1,float16,float16,0,7.784595489501953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,2,128,1,fp8,fp8,0,6.863191986083985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,4,128,1,float16,fp8,0,7.08880615234375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,2,128,1,float16,float16,0,8.179129791259765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,4,128,1,float16,float16,0,8.13551025390625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,16,128,1,float16,fp8,0,4.3413745880126955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,16,128,1,float16,float16,0,4.9187774658203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,1,128,1,float16,float16,0,3.8232688903808594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,16,128,1,fp8,fp8,0,4.378201675415039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,4,128,1,fp8,fp8,0,6.965427398681641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,8,128,1,float16,fp8,0,7.540265655517578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,8,128,1,fp8,fp8,0,7.545887756347656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,8,128,1,float16,float16,0,8.95467529296875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,1,128,1,float16,fp8,0,3.2881855010986327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,1,128,1,fp8,fp8,0,3.3160785675048827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,2,128,1,float16,fp8,0,3.381905746459961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,2,128,1,float16,float16,0,3.9696334838867187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,2,128,1,fp8,fp8,0,3.344188690185547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,4,128,1,float16,fp8,0,3.500414276123047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,4,128,1,float16,float16,0,4.0856689453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,4,128,1,fp8,fp8,0,3.5242366790771484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,16,128,1,float16,float16,0,2.4196271896362305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,8,128,1,float16,fp8,0,3.8836593627929688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,8,128,1,float16,float16,0,4.3849952697753904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,8,128,1,fp8,fp8,0,3.993172836303711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,16,128,1,float16,fp8,0,2.1818000793457033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,1,128,1,float16,float16,0,1.9022640228271483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,16,128,1,fp8,fp8,0,2.3333663940429688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,1,128,1,float16,fp8,0,1.7300527572631836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,1,128,1,fp8,fp8,0,1.6603151321411134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,2,128,1,float16,float16,0,1.8837520599365234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,2,128,1,float16,fp8,0,1.696873664855957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,2,128,1,fp8,fp8,0,1.6786752700805665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,4,128,1,float16,fp8,0,1.7971519470214843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,4,128,1,float16,float16,0,2.0775232315063477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,4,128,1,fp8,fp8,0,1.7997583389282226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,16,128,1,float16,float16,0,1.1968560218811035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,16,128,1,float16,fp8,0,1.1180496215820312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,8,128,1,float16,float16,0,2.178862380981445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,16,128,1,fp8,fp8,0,1.1132623672485351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,1,128,1,float16,float16,0,0.9688943862915039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,8,128,1,float16,fp8,0,1.930516815185547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,1,128,1,float16,fp8,0,0.845032024383545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,1,128,1,fp8,fp8,0,0.8789584159851074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,8,128,1,fp8,fp8,0,1.9085712432861328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,2,128,1,float16,float16,0,0.9281807899475097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,2,128,1,float16,fp8,0,0.9658335685729981
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,2,128,1,fp8,fp8,0,0.8628800392150879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,4,128,1,float16,float16,0,1.0528512001037598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,4,128,1,float16,fp8,0,0.8960720062255859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,4,128,1,fp8,fp8,0,0.897488021850586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,8,128,1,float16,float16,0,1.0461759567260742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,16,128,1,float16,float16,0,0.626643180847168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,8,128,1,float16,fp8,0,0.9671968460083008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,16,128,1,float16,fp8,0,0.5854496002197266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,1,128,1,float16,float16,0,0.4576128005981445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,16,128,1,fp8,fp8,0,0.6246064186096192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,1,128,1,float16,fp8,0,0.43646879196166993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,8,128,1,fp8,fp8,0,1.0070464134216308
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,1,128,1,fp8,fp8,0,0.4617136001586914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,2,128,1,float16,float16,0,0.4827871799468994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,2,128,1,float16,fp8,0,0.44929919242858884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,4,128,1,fp8,fp8,0,0.4624335765838623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,2,128,1,fp8,fp8,0,0.45594239234924316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,4,128,1,float16,float16,0,0.48725600242614747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,4,128,1,float16,fp8,0,0.4812592029571533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,8,128,1,float16,float16,0,0.5383887767791748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,8,128,1,float16,fp8,0,0.5121088027954102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,8,128,1,fp8,fp8,0,0.5106751918792725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,1,128,1,float16,fp8,0,3.901611328125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,1,128,1,fp8,fp8,0,3.823713684082031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,1,128,1,float16,float16,0,4.4280048370361325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,2,128,1,float16,fp8,0,3.943423843383789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,2,128,1,fp8,fp8,0,3.9965919494628905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,2,128,1,float16,float16,0,4.5307167053222654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,4,128,1,float16,fp8,0,4.191043090820313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,4,128,1,float16,float16,0,4.79759521484375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,1,128,1,float16,float16,0,2.1431343078613283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,16,128,1,float16,fp8,0,2.795694351196289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,16,128,1,float16,float16,0,3.0214719772338867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,4,128,1,fp8,fp8,0,4.141191864013672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,16,128,1,fp8,fp8,0,2.740020751953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,8,128,1,float16,fp8,0,4.607788848876953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,8,128,1,fp8,fp8,0,4.567547225952149
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,8,128,1,float16,float16,0,5.3197792053222654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,1,128,1,float16,fp8,0,2.0432416915893556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,1,128,1,fp8,fp8,0,1.92532958984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,2,128,1,float16,fp8,0,1.9557647705078125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,2,128,1,float16,float16,0,2.1906784057617186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,2,128,1,fp8,fp8,0,1.985910415649414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,4,128,1,float16,float16,0,2.3205503463745116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,4,128,1,float16,fp8,0,2.0781248092651365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,4,128,1,fp8,fp8,0,2.1950799942016603
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,16,128,1,float16,fp8,0,1.409715175628662
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,16,128,1,float16,float16,0,1.5759759902954102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,8,128,1,float16,fp8,0,2.2894208908081053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,1,128,1,float16,float16,0,1.0667152404785156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,8,128,1,float16,float16,0,2.5580207824707033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,1,128,1,float16,fp8,0,0.975380802154541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,8,128,1,fp8,fp8,0,2.2741439819335936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,16,128,1,fp8,fp8,0,1.5006848335266114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,1,128,1,fp8,fp8,0,0.9953184127807617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,2,128,1,float16,float16,0,1.0872127532958984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,2,128,1,float16,fp8,0,1.003934383392334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,2,128,1,fp8,fp8,0,1.0614591598510743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,4,128,1,float16,float16,0,1.1542783737182618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,4,128,1,float16,fp8,0,1.0575807571411133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,4,128,1,fp8,fp8,0,1.0466815948486328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,16,128,1,float16,float16,0,0.7798831939697266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,8,128,1,float16,float16,0,1.2561264038085938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,16,128,1,float16,fp8,0,0.7308191776275634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,8,128,1,float16,fp8,0,1.1843055725097655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,8,128,1,fp8,fp8,0,1.2013872146606446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,1,128,1,float16,float16,0,0.5288335800170898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,16,128,1,fp8,fp8,0,0.7783232212066651
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,1,128,1,float16,fp8,0,0.5065087795257568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,1,128,1,fp8,fp8,0,0.5093711853027344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,2,128,1,float16,float16,0,0.5840735912322998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,2,128,1,float16,fp8,0,0.5279151916503906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,2,128,1,fp8,fp8,0,0.5265120029449463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,4,128,1,float16,float16,0,0.5815983772277832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,4,128,1,float16,fp8,0,0.5746448040008545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,4,128,1,fp8,fp8,0,0.5553328037261963
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,8,128,1,float16,float16,0,0.6413104057312011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,8,128,1,float16,fp8,0,0.6208911895751953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,8,128,1,fp8,fp8,0,0.6062640190124512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,16,128,1,float16,float16,0,0.41611199378967284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,16,128,1,float16,fp8,0,0.377348804473877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,16,128,1,fp8,fp8,0,0.38726561069488524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,1,128,1,float16,float16,0,0.27703840732574464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,1,128,1,float16,fp8,0,0.26451520919799804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,2,128,1,fp8,fp8,0,0.27513439655303956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,1,128,1,fp8,fp8,0,0.27299520969390867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,2,128,1,float16,float16,0,0.28653440475463865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,2,128,1,float16,fp8,0,0.27569758892059326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,4,128,1,float16,float16,0,0.301363205909729
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,4,128,1,float16,fp8,0,0.29222400188446046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,4,128,1,fp8,fp8,0,0.2915791988372803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,8,128,1,float16,float16,0,0.3377232074737549
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,8,128,1,float16,fp8,0,0.3164495944976807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,8,128,1,fp8,fp8,0,0.318174409866333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,1,128,1,float16,fp8,0,3.6927310943603517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,1,128,1,fp8,fp8,0,3.665497589111328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,1,128,1,float16,float16,0,4.237756729125977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,2,128,1,float16,fp8,0,3.8321601867675783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,2,128,1,fp8,fp8,0,3.7753662109375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,2,128,1,float16,float16,0,4.338681411743164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,4,128,1,float16,float16,0,4.539457702636719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,16,128,1,float16,float16,0,3.1297328948974608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,4,128,1,float16,fp8,0,4.10599365234375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,4,128,1,fp8,fp8,0,4.05577278137207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,16,128,1,float16,fp8,0,3.056719970703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,16,128,1,fp8,fp8,0,3.00634880065918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,8,128,1,float16,fp8,0,4.62725601196289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,8,128,1,float16,float16,0,5.190406417846679
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,8,128,1,fp8,fp8,0,4.611835098266601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,1,128,1,float16,float16,0,2.01901912689209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,1,128,1,float16,fp8,0,1.8262016296386718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,1,128,1,fp8,fp8,0,1.8065568923950195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,2,128,1,float16,float16,0,2.062203216552734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,2,128,1,float16,fp8,0,1.897148895263672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,2,128,1,fp8,fp8,0,1.895782470703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,4,128,1,float16,fp8,0,2.03015193939209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,4,128,1,float16,float16,0,2.2347696304321287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,16,128,1,float16,float16,0,1.5753472328186036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,4,128,1,fp8,fp8,0,2.1553472518920898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,16,128,1,float16,fp8,0,1.4707167625427247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,8,128,1,float16,fp8,0,2.3056400299072264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,1,128,1,float16,float16,0,1.014889621734619
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,8,128,1,float16,float16,0,2.5271728515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,16,128,1,fp8,fp8,0,1.4598575592041017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,8,128,1,fp8,fp8,0,2.4942384719848634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,1,128,1,float16,fp8,0,0.939083194732666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,1,128,1,fp8,fp8,0,0.9456352233886719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,2,128,1,float16,float16,0,1.0381407737731934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,2,128,1,float16,fp8,0,0.9545136451721191
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,2,128,1,fp8,fp8,0,0.9977120399475098
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,4,128,1,float16,fp8,0,1.0275296211242675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,4,128,1,float16,float16,0,1.1352928161621094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,4,128,1,fp8,fp8,0,1.0594415664672852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,16,128,1,float16,float16,0,0.7860688209533692
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,8,128,1,float16,float16,0,1.2760255813598633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,16,128,1,float16,fp8,0,0.7677311897277832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,8,128,1,float16,fp8,0,1.1992544174194335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,8,128,1,fp8,fp8,0,1.258020782470703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,1,128,1,float16,float16,0,0.5201632022857666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,16,128,1,fp8,fp8,0,0.7490111827850342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,1,128,1,float16,fp8,0,0.4870143890380859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,1,128,1,fp8,fp8,0,0.47474079132080077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,2,128,1,float16,float16,0,0.5349967956542969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,2,128,1,float16,fp8,0,0.5054671764373779
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,2,128,1,fp8,fp8,0,0.4980735778808594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,4,128,1,float16,float16,0,0.5720511913299561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,4,128,1,float16,fp8,0,0.5477920055389405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,4,128,1,fp8,fp8,0,0.5382400035858155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,8,128,1,float16,float16,0,0.6373087882995605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,8,128,1,float16,fp8,0,0.6135471820831299
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,16,128,1,float16,float16,0,0.4136799812316895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,8,128,1,fp8,fp8,0,0.6140192031860352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,16,128,1,float16,fp8,0,0.38645598888397215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,16,128,1,fp8,fp8,0,0.3945487976074219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,1,128,1,float16,float16,0,0.2662031888961792
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,1,128,1,float16,fp8,0,0.2583872079849243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,1,128,1,fp8,fp8,0,0.25518240928649905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,2,128,1,float16,float16,0,0.2739599943161011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,2,128,1,float16,fp8,0,0.26747519969940187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,2,128,1,fp8,fp8,0,0.2670079946517944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,4,128,1,float16,float16,0,0.289849591255188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,4,128,1,float16,fp8,0,0.28355040550231936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,4,128,1,fp8,fp8,0,0.28578720092773435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,8,128,1,float16,float16,0,0.33321759700775144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,8,128,1,float16,fp8,0,0.31429119110107423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,8,128,1,fp8,fp8,0,0.3195024013519287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,1,128,1,fp8,fp8,0,0.13084640502929687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,16,128,1,float16,float16,0,0.22032480239868163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,16,128,1,float16,fp8,0,0.2088223934173584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,16,128,1,fp8,fp8,0,0.20437281131744384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,2,128,1,fp8,fp8,0,0.1386512041091919
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,1,128,1,float16,float16,0,0.1440608024597168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,1,128,1,float16,fp8,0,0.13072320222854614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,2,128,1,float16,float16,0,0.147544002532959
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,2,128,1,float16,fp8,0,0.13758399486541747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,4,128,1,float16,float16,0,0.15966399908065795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,4,128,1,float16,fp8,0,0.14450399875640868
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,4,128,1,fp8,fp8,0,0.14757440090179444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,8,128,1,float16,float16,0,0.18105599880218506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,8,128,1,float16,fp8,0,0.17051039934158324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,8,128,1,fp8,fp8,0,0.16829440593719483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,1,128,1,float16,float16,0,2.364017677307129
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,1,128,1,float16,fp8,0,2.2156112670898436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,1,128,1,fp8,fp8,0,2.217959976196289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,2,128,1,float16,float16,0,2.4105424880981445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,2,128,1,float16,fp8,0,2.328945541381836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,2,128,1,fp8,fp8,0,2.302547264099121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,4,128,1,float16,float16,0,2.7027055740356447
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,4,128,1,float16,fp8,0,2.665939140319824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,4,128,1,fp8,fp8,0,2.507289505004883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,16,128,1,float16,float16,0,2.0474767684936523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,16,128,1,float16,fp8,0,2.0276111602783202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,8,128,1,float16,float16,0,3.1822303771972655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,8,128,1,fp8,fp8,0,2.9499040603637696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,8,128,1,float16,fp8,0,3.038230323791504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,1,128,1,float16,float16,0,1.160976028442383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,1,128,1,float16,fp8,0,1.1007935523986816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,1,128,1,fp8,fp8,0,1.1327759742736816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,16,128,1,fp8,fp8,0,1.9563791275024414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,2,128,1,float16,float16,0,1.2230768203735352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,2,128,1,float16,fp8,0,1.231214427947998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,2,128,1,fp8,fp8,0,1.154811191558838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,4,128,1,float16,float16,0,1.3261743545532227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,4,128,1,float16,fp8,0,1.337003231048584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,4,128,1,fp8,fp8,0,1.2857407569885253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,16,128,1,float16,float16,0,1.0536160469055176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,8,128,1,float16,float16,0,1.581716823577881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,8,128,1,float16,fp8,0,1.4844719886779785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,16,128,1,float16,fp8,0,0.9837679862976074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,1,128,1,float16,float16,0,0.603268814086914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,1,128,1,float16,fp8,0,0.5753647804260253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,16,128,1,fp8,fp8,0,0.9848879814147949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,8,128,1,fp8,fp8,0,1.4993295669555664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,2,128,1,float16,fp8,0,0.5924895763397217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,1,128,1,fp8,fp8,0,0.5755695819854736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,2,128,1,float16,float16,0,0.6266960144042969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,2,128,1,fp8,fp8,0,0.5971903800964355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,4,128,1,float16,float16,0,0.6837535858154297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,4,128,1,float16,fp8,0,0.6574528217315674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,4,128,1,fp8,fp8,0,0.6641104221343994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,8,128,1,float16,float16,0,0.800318431854248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,8,128,1,float16,fp8,0,0.7527904033660888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,16,128,1,float16,float16,0,0.5435791969299316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,8,128,1,fp8,fp8,0,0.7684336185455323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,16,128,1,float16,fp8,0,0.5050320148468017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,1,128,1,float16,float16,0,0.31050078868865966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,16,128,1,fp8,fp8,0,0.5088191986083984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,1,128,1,float16,fp8,0,0.30379838943481446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,1,128,1,fp8,fp8,0,0.29996800422668457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,2,128,1,float16,float16,0,0.32006399631500243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,2,128,1,float16,fp8,0,0.31562879085540774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,2,128,1,fp8,fp8,0,0.31515839099884035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,4,128,1,float16,float16,0,0.3562400102615356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,4,128,1,float16,fp8,0,0.34301440715789794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,4,128,1,fp8,fp8,0,0.34168961048126223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,8,128,1,float16,float16,0,0.413267183303833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,8,128,1,float16,fp8,0,0.39445760250091555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,8,128,1,fp8,fp8,0,0.3946255922317505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,16,128,1,float16,float16,0,0.2869071960449219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,16,128,1,float16,fp8,0,0.26690399646759033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,16,128,1,fp8,fp8,0,0.26421759128570554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,1,128,1,float16,float16,0,0.16630879640579224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,1,128,1,float16,fp8,0,0.16361279487609864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,1,128,1,fp8,fp8,0,0.16201759576797486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,2,128,1,float16,float16,0,0.17414720058441163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,2,128,1,float16,fp8,0,0.1701248049736023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,2,128,1,fp8,fp8,0,0.16895359754562378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,4,128,1,float16,float16,0,0.18878560066223143
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,4,128,1,float16,fp8,0,0.18232959508895874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,4,128,1,fp8,fp8,0,0.184006404876709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,8,128,1,float16,float16,0,0.21954879760742188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,8,128,1,float16,fp8,0,0.21244161128997802
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,8,128,1,fp8,fp8,0,0.21028640270233154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,16,128,1,float16,float16,0,0.1560912013053894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,16,128,1,float16,fp8,0,0.14642239809036256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,16,128,1,fp8,fp8,0,0.14643360376358033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,1,128,1,float16,float16,0,0.09419040083885193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,1,128,1,float16,fp8,0,0.08931360244750977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,1,128,1,fp8,fp8,0,0.0891759991645813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,4,128,1,fp8,fp8,0,0.09763839840888977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,2,128,1,float16,float16,0,0.09714080095291137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,2,128,1,float16,fp8,0,0.09161760210990906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,2,128,1,fp8,fp8,0,0.09009439945220947
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,4,128,1,float16,float16,0,0.10820959806442261
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,4,128,1,float16,fp8,0,0.0984399974346161
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,8,128,1,float16,float16,0,0.12227519750595092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,8,128,1,float16,fp8,0,0.11237280368804932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,8,128,1,fp8,fp8,0,0.11324959993362427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,1,128,1,float16,float16,0,2.269419288635254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,1,128,1,float16,fp8,0,2.1855871200561525
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,1,128,1,fp8,fp8,0,2.170910453796387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,2,128,1,float16,float16,0,2.4616416931152343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,2,128,1,float16,fp8,0,2.372870445251465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,2,128,1,fp8,fp8,0,2.3186527252197267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,4,128,1,float16,float16,0,2.761252784729004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,4,128,1,float16,fp8,0,2.6777856826782225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,4,128,1,fp8,fp8,0,2.6031328201293946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,16,128,1,float16,float16,0,2.3102863311767576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,8,128,1,float16,float16,0,3.3804080963134764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,8,128,1,float16,fp8,0,3.288025665283203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,16,128,1,float16,fp8,0,2.261070442199707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,8,128,1,fp8,fp8,0,3.1677440643310546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,1,128,1,float16,float16,0,1.1294879913330078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,1,128,1,float16,fp8,0,1.109870433807373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,1,128,1,fp8,fp8,0,1.1334272384643556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,16,128,1,fp8,fp8,0,2.189164733886719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,2,128,1,float16,float16,0,1.2249135971069336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,2,128,1,float16,fp8,0,1.1881775856018066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,2,128,1,fp8,fp8,0,1.1557984352111816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,4,128,1,float16,float16,0,1.3752384185791016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,4,128,1,float16,fp8,0,1.325164794921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,4,128,1,fp8,fp8,0,1.3356224060058595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,16,128,1,float16,float16,0,1.1735679626464843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,8,128,1,float16,float16,0,1.6779136657714844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,16,128,1,float16,fp8,0,1.0937503814697265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,8,128,1,float16,fp8,0,1.6489408493041993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,1,128,1,float16,float16,0,0.5825136184692383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,8,128,1,fp8,fp8,0,1.603291130065918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,1,128,1,float16,fp8,0,0.5742303848266601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,16,128,1,fp8,fp8,0,1.1273167610168457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,1,128,1,fp8,fp8,0,0.5626160144805908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,2,128,1,float16,float16,0,0.6265215873718262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,2,128,1,float16,fp8,0,0.5917488098144531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,2,128,1,fp8,fp8,0,0.6082799911499024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,4,128,1,float16,float16,0,0.6998159885406494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,4,128,1,float16,fp8,0,0.6681920051574707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,4,128,1,fp8,fp8,0,0.6811344146728515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,8,128,1,float16,float16,0,0.8605487823486329
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,8,128,1,float16,fp8,0,0.815552043914795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,16,128,1,float16,float16,0,0.5995391845703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,8,128,1,fp8,fp8,0,0.8293871879577637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,16,128,1,float16,fp8,0,0.568828821182251
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,1,128,1,float16,float16,0,0.3081056118011475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,2,128,1,float16,fp8,0,0.31615519523620605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,16,128,1,fp8,fp8,0,0.5747424125671386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,1,128,1,float16,fp8,0,0.29526400566101074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,1,128,1,fp8,fp8,0,0.3003376007080078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,2,128,1,float16,float16,0,0.3216831922531128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,2,128,1,fp8,fp8,0,0.3187648057937622
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,4,128,1,float16,float16,0,0.3593839883804321
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,4,128,1,float16,fp8,0,0.35267999172210696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,4,128,1,fp8,fp8,0,0.3506464004516602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,8,128,1,float16,float16,0,0.44250240325927737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,8,128,1,float16,fp8,0,0.41992640495300293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,8,128,1,fp8,fp8,0,0.42728638648986816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,16,128,1,float16,float16,0,0.31480000019073484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,16,128,1,float16,fp8,0,0.3009119987487793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,16,128,1,fp8,fp8,0,0.2964063882827759
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,1,128,1,float16,float16,0,0.16792960166931153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,1,128,1,float16,fp8,0,0.16326719522476196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,4,128,1,float16,fp8,0,0.18748159408569337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,1,128,1,fp8,fp8,0,0.16447679996490477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,2,128,1,float16,float16,0,0.1728495955467224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,2,128,1,float16,fp8,0,0.17405920028686522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,2,128,1,fp8,fp8,0,0.17174559831619263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,4,128,1,float16,float16,0,0.1959280014038086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,4,128,1,fp8,fp8,0,0.19127680063247682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,8,128,1,float16,float16,0,0.234604811668396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,8,128,1,float16,fp8,0,0.22277441024780273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,8,128,1,fp8,fp8,0,0.22527039051055908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,16,128,1,float16,float16,0,0.1659664034843445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,16,128,1,float16,fp8,0,0.16292799711227418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,16,128,1,fp8,fp8,0,0.1613152027130127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,1,128,1,float16,float16,0,0.09049599766731262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,1,128,1,float16,fp8,0,0.08594880104064942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,1,128,1,fp8,fp8,0,0.08605120182037354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,2,128,1,float16,float16,0,0.09759520292282105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,8,128,1,float16,float16,0,0.12898240089416504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,2,128,1,float16,fp8,0,0.09200639724731445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,2,128,1,fp8,fp8,0,0.08996000289916992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,4,128,1,float16,float16,0,0.10873440504074097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,4,128,1,float16,fp8,0,0.10192639827728271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,4,128,1,fp8,fp8,0,0.10123679637908936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,8,128,1,float16,fp8,0,0.12407360076904297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,8,128,1,fp8,fp8,0,0.12519840002059937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,16,128,1,float16,float16,0,0.09340639710426331
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,16,128,1,float16,fp8,0,0.08409119844436645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,16,128,1,fp8,fp8,0,0.08432800173759461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,1,128,1,float16,float16,0,0.05223360061645508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,1,128,1,float16,fp8,0,0.05223199725151062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,1,128,1,fp8,fp8,0,0.05279200077056885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,2,128,1,float16,float16,0,0.0530128002166748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,2,128,1,float16,fp8,0,0.05343359708786011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,2,128,1,fp8,fp8,0,0.0532912015914917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,4,128,1,float16,float16,0,0.05883679986000061
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,4,128,1,float16,fp8,0,0.056785601377487185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,4,128,1,fp8,fp8,0,0.057550400495529175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,8,128,1,float16,float16,0,0.06905279755592346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,8,128,1,float16,fp8,0,0.06565600037574768
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,8,128,1,fp8,fp8,0,0.06484640240669251
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,1,128,1,float16,float16,0,1.400220775604248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,1,128,1,float16,fp8,0,1.3726847648620606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,1,128,1,fp8,fp8,0,1.3868960380554198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,2,128,1,float16,float16,0,1.5099136352539062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,2,128,1,float16,fp8,0,1.4878144264221191
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,2,128,1,fp8,fp8,0,1.510916805267334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,4,128,1,float16,float16,0,1.750067138671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,4,128,1,float16,fp8,0,1.7258367538452148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,4,128,1,fp8,fp8,0,1.6949024200439453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,8,128,1,float16,float16,0,2.2023103713989256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,16,128,1,float16,float16,0,1.5778783798217773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,8,128,1,float16,fp8,0,2.2050336837768554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,1,128,1,float16,float16,0,0.715934419631958
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,16,128,1,float16,fp8,0,1.5603952407836914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,8,128,1,fp8,fp8,0,2.1412336349487306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,16,128,1,fp8,fp8,0,1.5445759773254395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,1,128,1,float16,fp8,0,0.6979519844055175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,1,128,1,fp8,fp8,0,0.6991263866424561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,2,128,1,float16,float16,0,0.7784495830535889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,2,128,1,float16,fp8,0,0.7614031791687011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,2,128,1,fp8,fp8,0,0.765011215209961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,4,128,1,float16,float16,0,0.8870832443237304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,4,128,1,float16,fp8,0,0.8729392051696777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,4,128,1,fp8,fp8,0,0.8620368003845215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,8,128,1,float16,float16,0,1.1268431663513183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,8,128,1,float16,fp8,0,1.099127960205078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,1,128,1,float16,float16,0,0.3682912111282349
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,16,128,1,float16,float16,0,0.8125871658325196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,8,128,1,fp8,fp8,0,1.0943792343139649
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,16,128,1,float16,fp8,0,0.7837600231170654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,16,128,1,fp8,fp8,0,0.7872000217437745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,1,128,1,fp8,fp8,0,0.3682607889175415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,1,128,1,float16,fp8,0,0.3680783987045288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,2,128,1,float16,float16,0,0.3974656105041504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,2,128,1,float16,fp8,0,0.3965008020401001
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,2,128,1,fp8,fp8,0,0.3927743911743164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,4,128,1,float16,float16,0,0.4599279880523682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,4,128,1,float16,fp8,0,0.4462672233581543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,4,128,1,fp8,fp8,0,0.45033278465271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,8,128,1,float16,float16,0,0.5790512084960937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,8,128,1,float16,fp8,0,0.558897590637207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,8,128,1,fp8,fp8,0,0.5558688163757324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,16,128,1,float16,float16,0,0.4228400230407715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,16,128,1,float16,fp8,0,0.40563359260559084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,1,128,1,float16,float16,0,0.19852960109710693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,16,128,1,fp8,fp8,0,0.4063551902770996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,1,128,1,float16,fp8,0,0.19888639450073242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,1,128,1,fp8,fp8,0,0.19821759462356567
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,2,128,1,float16,float16,0,0.21264960765838622
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,2,128,1,float16,fp8,0,0.2109600067138672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,2,128,1,fp8,fp8,0,0.21172480583190917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,4,128,1,float16,float16,0,0.2423327922821045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,4,128,1,float16,fp8,0,0.23734240531921386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,4,128,1,fp8,fp8,0,0.2369215965270996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,8,128,1,float16,float16,0,0.3037168025970459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,8,128,1,float16,fp8,0,0.2946223974227905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,8,128,1,fp8,fp8,0,0.292526388168335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,16,128,1,float16,float16,0,0.2236112117767334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,16,128,1,float16,fp8,0,0.21699678897857666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,16,128,1,fp8,fp8,0,0.21401920318603515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,1,128,1,float16,float16,0,0.10980000495910644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,1,128,1,float16,fp8,0,0.11006400585174561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,1,128,1,fp8,fp8,0,0.1104464054107666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,2,128,1,float16,float16,0,0.11815359592437744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,2,128,1,float16,fp8,0,0.11621919870376587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,2,128,1,fp8,fp8,0,0.11719520092010498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,4,128,1,float16,float16,0,0.13280160427093507
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,4,128,1,float16,fp8,0,0.12964160442352296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,4,128,1,fp8,fp8,0,0.12955520153045655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,8,128,1,float16,float16,0,0.16313279867172242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,8,128,1,float16,fp8,0,0.1583616018295288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,8,128,1,fp8,fp8,0,0.1599184036254883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,16,128,1,float16,float16,0,0.12331199645996094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,16,128,1,float16,fp8,0,0.11802719831466675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,16,128,1,fp8,fp8,0,0.11881760358810425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,1,128,1,float16,float16,0,0.061211198568344116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,1,128,1,float16,fp8,0,0.06034560203552246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,1,128,1,fp8,fp8,0,0.06079360246658325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,2,128,1,float16,float16,0,0.06510239839553833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,2,128,1,float16,fp8,0,0.06326720118522644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,2,128,1,fp8,fp8,0,0.0631600022315979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,4,128,1,float16,float16,0,0.07561920285224914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,4,128,1,float16,fp8,0,0.0705407977104187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,4,128,1,fp8,fp8,0,0.07012479901313781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,8,128,1,float16,float16,0,0.09085440039634704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,8,128,1,float16,fp8,0,0.08528000116348267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,8,128,1,fp8,fp8,0,0.08738880157470703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,16,128,1,float16,float16,0,0.06884959936141968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,16,128,1,float16,fp8,0,0.061900800466537474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,16,128,1,fp8,fp8,0,0.061883199214935306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,1,128,1,float16,float16,0,0.041331198811531064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,1,128,1,float16,fp8,0,0.04115679860115051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,1,128,1,fp8,fp8,0,0.041038399934768675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,2,128,1,float16,float16,0,0.0419871985912323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,2,128,1,float16,fp8,0,0.04189920127391815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,2,128,1,fp8,fp8,0,0.042033600807189944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,4,128,1,float16,float16,0,0.04445120096206665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,4,128,1,float16,fp8,0,0.04469920098781586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,4,128,1,fp8,fp8,0,0.04503200054168701
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,8,128,1,float16,float16,0,0.05154399871826172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,8,128,1,float16,fp8,0,0.05042719841003418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,8,128,1,fp8,fp8,0,0.05034080147743225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,1,128,1,float16,float16,0,1.4717280387878418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,1,128,1,float16,fp8,0,1.460484790802002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,1,128,1,fp8,fp8,0,1.4516351699829102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,2,128,1,float16,float16,0,1.6162351608276366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,2,128,1,float16,fp8,0,1.5976351737976073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,2,128,1,fp8,fp8,0,1.5947967529296876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,4,128,1,float16,float16,0,1.9343391418457032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,4,128,1,float16,fp8,0,1.909529685974121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,4,128,1,fp8,fp8,0,1.8807167053222655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,8,128,1,float16,float16,0,2.549929618835449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,1,128,1,float16,float16,0,0.7349775791168213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,16,128,1,float16,float16,0,1.9188735961914063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,8,128,1,float16,fp8,0,2.4642192840576174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,16,128,1,float16,fp8,0,1.8415103912353517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,8,128,1,fp8,fp8,0,2.4768159866333006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,16,128,1,fp8,fp8,0,1.8696144104003907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,1,128,1,fp8,fp8,0,0.7380959987640381
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,1,128,1,float16,fp8,0,0.7462687969207764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,2,128,1,float16,float16,0,0.8249135971069336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,2,128,1,float16,fp8,0,0.8178832054138183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,2,128,1,fp8,fp8,0,0.8088255882263183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,4,128,1,float16,float16,0,0.9676480293273926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,4,128,1,float16,fp8,0,0.95425443649292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,4,128,1,fp8,fp8,0,0.9520959854125977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,8,128,1,float16,float16,0,1.2993231773376466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,16,128,1,float16,float16,0,0.9848095893859863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,8,128,1,float16,fp8,0,1.257044792175293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,1,128,1,float16,float16,0,0.39383199214935305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,16,128,1,float16,fp8,0,0.9324576377868652
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,16,128,1,fp8,fp8,0,0.9337632179260253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,1,128,1,float16,fp8,0,0.3866559982299805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,8,128,1,fp8,fp8,0,1.2602288246154785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,1,128,1,fp8,fp8,0,0.38951361179351807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,2,128,1,float16,float16,0,0.42366719245910645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,2,128,1,float16,fp8,0,0.4259471893310547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,2,128,1,fp8,fp8,0,0.41698880195617677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,4,128,1,float16,float16,0,0.4981872081756592
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,4,128,1,float16,fp8,0,0.4927519798278809
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,4,128,1,fp8,fp8,0,0.49262399673461915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,8,128,1,float16,float16,0,0.6674767971038819
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,8,128,1,float16,fp8,0,0.640399980545044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,8,128,1,fp8,fp8,0,0.6477888107299805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,16,128,1,float16,float16,0,0.5037360191345215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,1,128,1,float16,float16,0,0.20814878940582277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,16,128,1,float16,fp8,0,0.47966718673706055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,1,128,1,float16,fp8,0,0.20306079387664794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,16,128,1,fp8,fp8,0,0.48438239097595215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,1,128,1,fp8,fp8,0,0.20635519027709961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,2,128,1,float16,float16,0,0.22665278911590575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,2,128,1,float16,fp8,0,0.22522239685058593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,2,128,1,fp8,fp8,0,0.224019193649292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,4,128,1,float16,float16,0,0.266051197052002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,4,128,1,float16,fp8,0,0.2607439994812012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,4,128,1,fp8,fp8,0,0.25586719512939454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,8,128,1,float16,float16,0,0.34528961181640627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,8,128,1,float16,fp8,0,0.3339279890060425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,8,128,1,fp8,fp8,0,0.33181118965148926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,16,128,1,float16,float16,0,0.26563520431518556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,2,128,1,float16,float16,0,0.12653759717941285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,16,128,1,float16,fp8,0,0.25367040634155275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,1,128,1,float16,float16,0,0.1175663948059082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,16,128,1,fp8,fp8,0,0.2540816068649292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,1,128,1,float16,fp8,0,0.11471519470214844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,1,128,1,fp8,fp8,0,0.11700639724731446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,2,128,1,float16,fp8,0,0.12438880205154419
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,2,128,1,fp8,fp8,0,0.1256224036216736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,4,128,1,float16,float16,0,0.14649759531021117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,4,128,1,float16,fp8,0,0.14276959896087646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,4,128,1,fp8,fp8,0,0.14217280149459838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,8,128,1,float16,float16,0,0.1852944016456604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,8,128,1,float16,fp8,0,0.17640960216522217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,8,128,1,fp8,fp8,0,0.17857760190963745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,16,128,1,float16,float16,0,0.1429263949394226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,16,128,1,float16,fp8,0,0.13969919681549073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,16,128,1,fp8,fp8,0,0.13742560148239136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,1,128,1,float16,float16,0,0.06779680252075196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,1,128,1,float16,fp8,0,0.06302880048751831
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,1,128,1,fp8,fp8,0,0.06406559944152831
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,4,128,1,fp8,fp8,0,0.07787680029869079
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,2,128,1,float16,float16,0,0.07323200106620789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,2,128,1,float16,fp8,0,0.067958402633667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,2,128,1,fp8,fp8,0,0.06902559995651245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,4,128,1,float16,float16,0,0.08263840079307556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,4,128,1,float16,fp8,0,0.07783520221710205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,8,128,1,float16,float16,0,0.10257920026779174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,8,128,1,float16,fp8,0,0.10047999620437623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,8,128,1,fp8,fp8,0,0.10036959648132324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,16,128,1,float16,float16,0,0.08029919862747192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,16,128,1,float16,fp8,0,0.07150880098342896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,16,128,1,fp8,fp8,0,0.07342079877853394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,1,128,1,float16,float16,0,0.039719998836517334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,1,128,1,float16,fp8,0,0.040803200006484984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,1,128,1,fp8,fp8,0,0.040996798872947694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,2,128,1,float16,float16,0,0.040987199544906615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,2,128,1,float16,fp8,0,0.04195840060710907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,2,128,1,fp8,fp8,0,0.04175840020179748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,4,128,1,float16,float16,0,0.0460752010345459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,4,128,1,float16,fp8,0,0.046823999285697936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,4,128,1,fp8,fp8,0,0.04589279890060425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,8,128,1,float16,float16,0,0.05607039928436279
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,8,128,1,float16,fp8,0,0.053364801406860354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,8,128,1,fp8,fp8,0,0.05393760204315186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,16,128,1,float16,float16,0,0.04733920097351074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,16,128,1,float16,fp8,0,0.047830399870872495
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,16,128,1,fp8,fp8,0,0.04806239902973175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,2,128,1,fp8,fp8,0,0.03515680134296417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,1,128,1,float16,float16,0,0.03452160060405731
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,1,128,1,float16,fp8,0,0.03531999886035919
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,1,128,1,fp8,fp8,0,0.034883201122283936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,2,128,1,float16,float16,0,0.03493599891662598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,2,128,1,float16,fp8,0,0.036268800497055054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,4,128,1,float16,float16,0,0.035499200224876404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,4,128,1,float16,fp8,0,0.03599520027637482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,4,128,1,fp8,fp8,0,0.036006399989128114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,8,128,1,float16,float16,0,0.04035359919071198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,8,128,1,float16,fp8,0,0.04038879871368408
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,8,128,1,fp8,fp8,0,0.04079360067844391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,1,128,1,float16,float16,0,1.0712384223937987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,1,128,1,float16,fp8,0,1.0917967796325683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,1,128,1,fp8,fp8,0,1.0948863983154298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,2,128,1,float16,float16,0,1.2254863739013673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,2,128,1,float16,fp8,0,1.2447152137756348
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,2,128,1,fp8,fp8,0,1.2356287956237793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,4,128,1,float16,float16,0,1.5479040145874023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,4,128,1,float16,fp8,0,1.5175600051879883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,4,128,1,fp8,fp8,0,1.5239952087402344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,8,128,1,float16,float16,0,2.1620784759521485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,1,128,1,float16,float16,0,0.5482528209686279
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,16,128,1,float16,float16,0,1.71551513671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,8,128,1,float16,fp8,0,2.1249584197998046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,8,128,1,fp8,fp8,0,2.118179130554199
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,16,128,1,float16,fp8,0,1.6850751876831054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,1,128,1,float16,fp8,0,0.5606847763061523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,16,128,1,fp8,fp8,0,1.6699520111083985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,1,128,1,fp8,fp8,0,0.5562704086303711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,2,128,1,float16,float16,0,0.6261295795440673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,2,128,1,float16,fp8,0,0.628326416015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,2,128,1,fp8,fp8,0,0.6297808170318604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,4,128,1,float16,float16,0,0.7776976108551026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,4,128,1,float16,fp8,0,0.7704448223114013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,4,128,1,fp8,fp8,0,0.7729455947875976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,8,128,1,float16,float16,0,1.086907196044922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,8,128,1,float16,fp8,0,1.0697487831115722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,1,128,1,float16,float16,0,0.28612000942230226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,16,128,1,float16,float16,0,0.8796159744262695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,16,128,1,fp8,fp8,0,0.8571439743041992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,8,128,1,fp8,fp8,0,1.0810352325439454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,1,128,1,float16,fp8,0,0.29280641078948977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,2,128,1,float16,fp8,0,0.3266448020935059
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,16,128,1,float16,fp8,0,0.8443327903747558
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,1,128,1,fp8,fp8,0,0.29193921089172364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,2,128,1,float16,float16,0,0.32361760139465334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,2,128,1,fp8,fp8,0,0.330348801612854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,4,128,1,float16,float16,0,0.4066768169403076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,4,128,1,float16,fp8,0,0.40114078521728513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,4,128,1,fp8,fp8,0,0.3982880115509033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,8,128,1,float16,float16,0,0.5579664230346679
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,8,128,1,float16,fp8,0,0.5449823856353759
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,8,128,1,fp8,fp8,0,0.5466527938842773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,16,128,1,float16,float16,0,0.4528975963592529
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,16,128,1,float16,fp8,0,0.4407792091369629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,1,128,1,float16,float16,0,0.15615520477294922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,16,128,1,fp8,fp8,0,0.44025440216064454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,1,128,1,float16,fp8,0,0.1586143970489502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,4,128,1,float16,fp8,0,0.2113503932952881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,1,128,1,fp8,fp8,0,0.1584879994392395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,2,128,1,float16,float16,0,0.1744223952293396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,2,128,1,float16,fp8,0,0.1761023998260498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,2,128,1,fp8,fp8,0,0.17885279655456543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,4,128,1,float16,float16,0,0.21508960723876952
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,4,128,1,fp8,fp8,0,0.2130687952041626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,8,128,1,float16,float16,0,0.2910592079162598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,8,128,1,float16,fp8,0,0.28552000522613524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,8,128,1,fp8,fp8,0,0.2849616050720215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,16,128,1,float16,float16,0,0.2366544008255005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,16,128,1,float16,fp8,0,0.22864480018615724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,2,128,1,float16,fp8,0,0.10065439939498902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,16,128,1,fp8,fp8,0,0.23257920742034913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,1,128,1,float16,float16,0,0.09219520092010498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,1,128,1,float16,fp8,0,0.09385759830474853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,1,128,1,fp8,fp8,0,0.09311040043830872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,2,128,1,float16,float16,0,0.09942399859428405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,2,128,1,fp8,fp8,0,0.1007200002670288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,4,128,1,float16,float16,0,0.11794079542160034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,4,128,1,float16,fp8,0,0.11821600198745727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,4,128,1,fp8,fp8,0,0.12040959596633911
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,8,128,1,float16,float16,0,0.15865600109100342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,8,128,1,float16,fp8,0,0.15612319707870484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,8,128,1,fp8,fp8,0,0.15518720149993898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,16,128,1,float16,float16,0,0.1299631953239441
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,16,128,1,float16,fp8,0,0.1284432053565979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,16,128,1,fp8,fp8,0,0.12762720584869386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,1,128,1,float16,float16,0,0.05543199777603149
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,1,128,1,float16,fp8,0,0.05358080267906189
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,1,128,1,fp8,fp8,0,0.052014398574829104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,2,128,1,float16,float16,0,0.0631376028060913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,2,128,1,float16,fp8,0,0.058064001798629764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,2,128,1,fp8,fp8,0,0.05715680122375488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,4,128,1,float16,float16,0,0.07125279903411866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,4,128,1,float16,fp8,0,0.06832799911499024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,4,128,1,fp8,fp8,0,0.06784160137176513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,8,128,1,float16,float16,0,0.09190400242805481
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,8,128,1,float16,fp8,0,0.08903840184211731
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,8,128,1,fp8,fp8,0,0.08811519742012024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,16,128,1,float16,float16,0,0.07671840190887451
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,16,128,1,float16,fp8,0,0.0693615972995758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,16,128,1,fp8,fp8,0,0.06886240243911743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,1,128,1,float16,float16,0,0.03598879873752594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,1,128,1,float16,fp8,0,0.03701280057430267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,1,128,1,fp8,fp8,0,0.037099200487136844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,2,128,1,float16,float16,0,0.036606401205062866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,2,128,1,float16,fp8,0,0.03794879913330078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,2,128,1,fp8,fp8,0,0.03765920102596283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,4,128,1,float16,float16,0,0.04187999963760376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,4,128,1,float16,fp8,0,0.04247680008411407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,4,128,1,fp8,fp8,0,0.04240640103816986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,8,128,1,float16,float16,0,0.052219200134277347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,8,128,1,float16,fp8,0,0.049667200446128844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,8,128,1,fp8,fp8,0,0.04954560101032257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,16,128,1,float16,float16,0,0.04207200109958649
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,16,128,1,float16,fp8,0,0.04264959990978241
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,16,128,1,fp8,fp8,0,0.042345601320266726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,1,128,1,float16,float16,0,0.02855519950389862
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,1,128,1,float16,fp8,0,0.029783999919891356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,1,128,1,fp8,fp8,0,0.029552000761032104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,2,128,1,float16,float16,0,0.028646400570869444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,2,128,1,float16,fp8,0,0.029702401161193846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,2,128,1,fp8,fp8,0,0.029891198873519896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,4,128,1,float16,float16,0,0.029691201448440552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,4,128,1,float16,fp8,0,0.030983999371528625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,4,128,1,fp8,fp8,0,0.030908799171447753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,8,128,1,float16,float16,0,0.0341728001832962
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,8,128,1,float16,fp8,0,0.03517760038375854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,8,128,1,fp8,fp8,0,0.03524479866027832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,16,128,1,float16,float16,0,0.033406400680541994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,16,128,1,float16,fp8,0,0.03496159911155701
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,16,128,1,fp8,fp8,0,0.03522560000419617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,1,128,1,float16,float16,0,0.027750399708747864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,1,128,1,float16,fp8,0,0.029683199524879456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,1,128,1,fp8,fp8,0,0.02927359938621521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,2,128,1,float16,float16,0,0.02802720069885254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,2,128,1,float16,fp8,0,0.02954080104827881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,4,128,1,float16,float16,0,0.028428798913955687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,2,128,1,fp8,fp8,0,0.0295199990272522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,4,128,1,float16,fp8,0,0.029507198929786684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,4,128,1,fp8,fp8,0,0.029422399401664735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,8,128,1,float16,float16,0,0.029014399647712706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,8,128,1,float16,fp8,0,0.030579200387001036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,8,128,1,fp8,fp8,0,0.030516800284385682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,1,128,1,float16,float16,0,0.45947041511535647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,1,128,1,float16,fp8,0,0.47907519340515137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,1,128,1,fp8,fp8,0,0.4751440048217773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,2,128,1,float16,float16,0,0.5411407947540283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,2,128,1,float16,fp8,0,0.5500207901000976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,2,128,1,fp8,fp8,0,0.5473696231842041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,4,128,1,float16,float16,0,0.693071985244751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,4,128,1,float16,fp8,0,0.6950096130371094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,4,128,1,fp8,fp8,0,0.6914144039154053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,8,128,1,float16,float16,0,1.0119839668273927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,8,128,1,float16,fp8,0,0.9933759689331054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,1,128,1,float16,float16,0,0.2447376012802124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,16,128,1,float16,float16,0,0.8420559883117675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,8,128,1,fp8,fp8,0,0.9978287696838379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,1,128,1,float16,fp8,0,0.251198410987854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,16,128,1,float16,fp8,0,0.7981488227844238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,1,128,1,fp8,fp8,0,0.2523855924606323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,16,128,1,fp8,fp8,0,0.7977168083190918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,2,128,1,float16,float16,0,0.28217599391937254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,2,128,1,float16,fp8,0,0.28815360069274903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,2,128,1,fp8,fp8,0,0.285531210899353
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,4,128,1,float16,float16,0,0.3620800018310547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,4,128,1,float16,fp8,0,0.35577759742736814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,4,128,1,fp8,fp8,0,0.3565104007720947
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,8,128,1,float16,float16,0,0.5199615955352783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,8,128,1,float16,fp8,0,0.5079216003417969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,8,128,1,fp8,fp8,0,0.5084527969360352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,16,128,1,float16,float16,0,0.4316847801208496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,1,128,1,float16,float16,0,0.1336143970489502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,2,128,1,float16,fp8,0,0.1562831997871399
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,16,128,1,float16,fp8,0,0.4093696117401123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,16,128,1,fp8,fp8,0,0.40946078300476074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,1,128,1,float16,fp8,0,0.13823039531707765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,1,128,1,fp8,fp8,0,0.13888319730758666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,2,128,1,float16,float16,0,0.1531440019607544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,2,128,1,fp8,fp8,0,0.1557423949241638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,4,128,1,float16,float16,0,0.19144320487976074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,4,128,1,float16,fp8,0,0.19056479930877684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,4,128,1,fp8,fp8,0,0.19057120084762574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,8,128,1,float16,float16,0,0.2704319953918457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,8,128,1,float16,fp8,0,0.26603360176086427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,8,128,1,fp8,fp8,0,0.2669008016586304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,16,128,1,float16,float16,0,0.22882239818572997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,16,128,1,float16,fp8,0,0.2167135953903198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,16,128,1,fp8,fp8,0,0.21634399890899658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,2,128,1,fp8,fp8,0,0.09040799736976624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,1,128,1,float16,float16,0,0.08097280263900757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,1,128,1,float16,fp8,0,0.08268960118293762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,1,128,1,fp8,fp8,0,0.0825872004032135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,2,128,1,float16,float16,0,0.08952959775924682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,2,128,1,float16,fp8,0,0.09055359959602356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,4,128,1,float16,float16,0,0.10848959684371948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,4,128,1,float16,fp8,0,0.10772000551223755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,4,128,1,fp8,fp8,0,0.10790400505065918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,16,128,1,float16,fp8,0,0.1194607973098755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,8,128,1,float16,float16,0,0.14772800207138062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,8,128,1,float16,fp8,0,0.14499839544296264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,8,128,1,fp8,fp8,0,0.14567359685897827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,16,128,1,float16,float16,0,0.12676000595092773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,16,128,1,fp8,fp8,0,0.11993119716644288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,1,128,1,float16,float16,0,0.04896480143070221
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,1,128,1,float16,fp8,0,0.046081599593162534
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,1,128,1,fp8,fp8,0,0.04737600088119507
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,2,128,1,float16,float16,0,0.056327998638153076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,2,128,1,float16,fp8,0,0.053830397129058835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,2,128,1,fp8,fp8,0,0.05275200009346008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,4,128,1,float16,float16,0,0.06503199934959411
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,4,128,1,float16,fp8,0,0.061780798435211184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,4,128,1,fp8,fp8,0,0.06182399988174438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,8,128,1,float16,float16,0,0.08576639890670776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,8,128,1,float16,fp8,0,0.08299840092658997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,8,128,1,fp8,fp8,0,0.08340799808502197
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,16,128,1,float16,float16,0,0.07294880151748658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,16,128,1,float16,fp8,0,0.06314399838447571
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,16,128,1,fp8,fp8,0,0.06233599781990051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,1,128,1,float16,float16,0,0.03160960078239441
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,1,128,1,float16,fp8,0,0.03352479934692383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,1,128,1,fp8,fp8,0,0.03332639932632446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,2,128,1,float16,float16,0,0.03249920010566711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,2,128,1,float16,fp8,0,0.03434720039367676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,2,128,1,fp8,fp8,0,0.0343423992395401
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,4,128,1,float16,float16,0,0.037062400579452516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,4,128,1,float16,fp8,0,0.03849120140075683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,4,128,1,fp8,fp8,0,0.03840480148792267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,8,128,1,float16,float16,0,0.048102399706840514
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,16,128,1,fp8,fp8,0,0.038657599687576295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,8,128,1,float16,fp8,0,0.046060800552368164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,8,128,1,fp8,fp8,0,0.04661599993705749
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,16,128,1,float16,float16,0,0.03893760144710541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,16,128,1,float16,fp8,0,0.038689601421356204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,1,128,1,float16,float16,0,0.02518880069255829
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,1,128,1,float16,fp8,0,0.025889599323272706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,1,128,1,fp8,fp8,0,0.026206400990486146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,2,128,1,float16,float16,0,0.02573919892311096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,2,128,1,float16,fp8,0,0.026209598779678343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,2,128,1,fp8,fp8,0,0.02630879878997803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,4,128,1,float16,float16,0,0.026395198702812196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,4,128,1,float16,fp8,0,0.027134400606155396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,4,128,1,fp8,fp8,0,0.027105599641799927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,8,128,1,float16,float16,0,0.03080959916114807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,8,128,1,float16,fp8,0,0.03147999942302704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,8,128,1,fp8,fp8,0,0.03155680000782013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,16,128,1,float16,float16,0,0.030420801043510436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,16,128,1,float16,fp8,0,0.031563198566436766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,16,128,1,fp8,fp8,0,0.03156799972057343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,1,128,1,float16,float16,0,0.025216001272201537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,1,128,1,float16,fp8,0,0.025916799902915955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,1,128,1,fp8,fp8,0,0.025835201144218445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,2,128,1,float16,float16,0,0.024977600574493407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,2,128,1,float16,fp8,0,0.02574560046195984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,2,128,1,fp8,fp8,0,0.025814399123191833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,4,128,1,float16,float16,0,0.025593599677085875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,4,128,1,float16,fp8,0,0.025995200872421263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,4,128,1,fp8,fp8,0,0.02632479965686798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,8,128,1,float16,float16,0,0.02617279887199402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,8,128,1,float16,fp8,0,0.026764801144599913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,8,128,1,fp8,fp8,0,0.026980799436569215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,16,128,1,float16,float16,0,0.025009599328041077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,16,128,1,float16,fp8,0,0.025696000456809996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,16,128,1,fp8,fp8,0,0.026004800200462343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,1,128,1,float16,float16,0,0.023396800458431243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,1,128,1,float16,fp8,0,0.02454880028963089
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,1,128,1,fp8,fp8,0,0.024382400512695312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,2,128,1,float16,float16,0,0.02359839975833893
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,2,128,1,float16,fp8,0,0.024585600197315215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,2,128,1,fp8,fp8,0,0.02494720071554184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,4,128,1,float16,float16,0,0.023758399486541747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,4,128,1,float16,fp8,0,0.024539199471473695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,4,128,1,fp8,fp8,0,0.0247856006026268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,8,128,1,float16,float16,0,0.023998400568962096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,8,128,1,float16,fp8,0,0.024961599707603456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,8,128,1,fp8,fp8,0,0.024728000164031982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,1,128,1,float16,float16,0,0.23551199436187745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,1,128,1,float16,fp8,0,0.24877281188964845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,1,128,1,fp8,fp8,0,0.24753119945526122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,2,128,1,float16,float16,0,0.27502079010009767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,2,128,1,float16,fp8,0,0.28667678833007815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,2,128,1,fp8,fp8,0,0.28609280586242675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,4,128,1,float16,float16,0,0.3537600040435791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,4,128,1,float16,fp8,0,0.3608319997787476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,4,128,1,fp8,fp8,0,0.3613055944442749
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,8,128,1,float16,float16,0,0.517580795288086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,8,128,1,float16,fp8,0,0.5078576087951661
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,8,128,1,fp8,fp8,0,0.5110223770141602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,16,128,1,float16,float16,0,0.4341760158538818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,16,128,1,float16,fp8,0,0.41393442153930665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,1,128,1,float16,float16,0,0.13044639825820922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,16,128,1,fp8,fp8,0,0.41263680458068847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,1,128,1,float16,fp8,0,0.13647840023040772
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,1,128,1,fp8,fp8,0,0.13666239976882935
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,2,128,1,float16,float16,0,0.14869439601898193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,2,128,1,float16,fp8,0,0.15698879957199097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,2,128,1,fp8,fp8,0,0.15594719648361205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,4,128,1,float16,float16,0,0.18880159854888917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,8,128,1,float16,fp8,0,0.2653775930404663
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,4,128,1,float16,fp8,0,0.1934432029724121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,4,128,1,fp8,fp8,0,0.19288320541381837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,8,128,1,float16,float16,0,0.26966559886932373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,8,128,1,fp8,fp8,0,0.26680641174316405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,16,128,1,float16,float16,0,0.23049600124359132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,16,128,1,float16,fp8,0,0.21619679927825927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,1,128,1,float16,float16,0,0.07929760217666626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,16,128,1,fp8,fp8,0,0.2165855884552002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,1,128,1,float16,fp8,0,0.08075839877128602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,1,128,1,fp8,fp8,0,0.08095679879188537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,2,128,1,float16,float16,0,0.08748000264167785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,2,128,1,float16,fp8,0,0.08900160193443299
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,2,128,1,fp8,fp8,0,0.08875679969787598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,4,128,1,float16,float16,0,0.1068608045578003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,4,128,1,float16,fp8,0,0.10700639486312866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,4,128,1,fp8,fp8,0,0.10686880350112915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,8,128,1,float16,float16,0,0.14682400226593018
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,8,128,1,float16,fp8,0,0.14277440309524536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,8,128,1,fp8,fp8,0,0.14284000396728516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,16,128,1,float16,float16,0,0.12545119524002074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,16,128,1,float16,fp8,0,0.11718080043792725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,16,128,1,fp8,fp8,0,0.11726720333099365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,1,128,1,float16,float16,0,0.044233599305152894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,1,128,1,float16,fp8,0,0.04187999963760376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,1,128,1,fp8,fp8,0,0.04380159974098206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,2,128,1,float16,float16,0,0.0532800018787384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,2,128,1,float16,fp8,0,0.04775519967079163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,2,128,1,fp8,fp8,0,0.04839200079441071
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,4,128,1,float16,float16,0,0.06357759833335877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,4,128,1,float16,fp8,0,0.057183998823165896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,4,128,1,fp8,fp8,0,0.05721120238304138
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,8,128,1,float16,float16,0,0.08372799754142761
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,8,128,1,float16,fp8,0,0.07998080253601074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,8,128,1,fp8,fp8,0,0.07929440140724182
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,16,128,1,float16,float16,0,0.07047200202941895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,2,128,1,float16,float16,0,0.030019199848175047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,16,128,1,float16,fp8,0,0.0599232017993927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,16,128,1,fp8,fp8,0,0.05872640013694763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,1,128,1,float16,float16,0,0.02922559976577759
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,1,128,1,float16,fp8,0,0.029993599653244017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,1,128,1,fp8,fp8,0,0.030140799283981324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,2,128,1,float16,fp8,0,0.03096800148487091
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,2,128,1,fp8,fp8,0,0.03077920079231262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,4,128,1,float16,float16,0,0.03531999886035919
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,4,128,1,float16,fp8,0,0.03505600094795227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,4,128,1,fp8,fp8,0,0.03529280126094818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,8,128,1,float16,float16,0,0.04511680006980896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,8,128,1,float16,fp8,0,0.04246560037136078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,8,128,1,fp8,fp8,0,0.04301120042800903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,16,128,1,float16,float16,0,0.037231999635696414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,16,128,1,float16,fp8,0,0.036796799302101134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,16,128,1,fp8,fp8,0,0.03700959980487824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,1,128,1,float16,float16,0,0.023841600120067596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,1,128,1,float16,fp8,0,0.02451840043067932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,1,128,1,fp8,fp8,0,0.02452320009469986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,2,128,1,float16,float16,0,0.02417600005865097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,2,128,1,float16,fp8,0,0.02478879988193512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,2,128,1,fp8,fp8,0,0.02489439994096756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,4,128,1,float16,float16,0,0.02507840096950531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,4,128,1,float16,fp8,0,0.025681599974632263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,4,128,1,fp8,fp8,0,0.02571519911289215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,8,128,1,float16,float16,0,0.029552000761032104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,8,128,1,float16,fp8,0,0.029790401458740234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,8,128,1,fp8,fp8,0,0.029974400997161865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,16,128,1,float16,float16,0,0.029009601473808287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,16,128,1,float16,fp8,0,0.029238399863243104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,16,128,1,fp8,fp8,0,0.02951039969921112
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,1,128,1,float16,float16,0,0.023384000360965728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,1,128,1,float16,fp8,0,0.024243199825286867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,1,128,1,fp8,fp8,0,0.024438400566577912
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,2,128,1,float16,float16,0,0.023528000712394713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,2,128,1,float16,fp8,0,0.024340799450874327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,2,128,1,fp8,fp8,0,0.02420479953289032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,4,128,1,float16,float16,0,0.02388000041246414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,4,128,1,float16,fp8,0,0.024182400107383727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,4,128,1,fp8,fp8,0,0.02415200024843216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,8,128,1,float16,float16,0,0.024612799286842346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,8,128,1,float16,fp8,0,0.024984000623226164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,8,128,1,fp8,fp8,0,0.02526719868183136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,16,128,1,float16,float16,0,0.023499199748039247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,16,128,1,float16,fp8,0,0.02396959960460663
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,16,128,1,fp8,fp8,0,0.024040000140666963
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,4,128,1,float16,float16,0,0.022169600427150726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,1,128,1,float16,float16,0,0.02208160012960434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,1,128,1,float16,fp8,0,0.022771200537681578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,1,128,1,fp8,fp8,0,0.022758400440216063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,2,128,1,float16,float16,0,0.02220800071954727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,2,128,1,float16,fp8,0,0.022867199778556824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,2,128,1,fp8,fp8,0,0.023219199478626253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,4,128,1,float16,fp8,0,0.022603200376033784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,4,128,1,fp8,fp8,0,0.022894400358200073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,8,128,1,float16,float16,0,0.022518399357795715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,8,128,1,float16,fp8,0,0.0234607994556427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,8,128,1,fp8,fp8,0,0.02293439954519272
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,1,128,1,fp8,fp8,0,0.02179519981145859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,16,128,1,float16,float16,0,0.021459199488162994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,16,128,1,float16,fp8,0,0.022065599262714387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,16,128,1,fp8,fp8,0,0.02232320010662079
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,1,128,1,float16,float16,0,0.02088160067796707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,1,128,1,float16,fp8,0,0.02165440022945404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,4,128,1,fp8,fp8,0,0.021753600239753722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,2,128,1,float16,float16,0,0.020897600054740905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,8,128,1,float16,fp8,0,0.021721599996089934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,2,128,1,float16,fp8,0,0.021913599967956544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,2,128,1,fp8,fp8,0,0.021972799301147462
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,4,128,1,float16,float16,0,0.02141920030117035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,1,128,1,float16,fp8,0,0.1377776026725769
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,4,128,1,float16,fp8,0,0.02194879949092865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,8,128,1,float16,float16,0,0.021367999911308288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,8,128,1,fp8,fp8,0,0.02176479995250702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,1,128,1,float16,float16,0,0.13063039779663085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,1,128,1,fp8,fp8,0,0.13793280124664306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,2,128,1,float16,float16,0,0.15001920461654664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,2,128,1,float16,fp8,0,0.15674079656600953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,2,128,1,fp8,fp8,0,0.15584800243377686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,4,128,1,float16,float16,0,0.18996959924697876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,4,128,1,float16,fp8,0,0.1931424021720886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,4,128,1,fp8,fp8,0,0.19317280054092406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,8,128,1,float16,float16,0,0.2716016054153442
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,8,128,1,float16,fp8,0,0.26578559875488283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,1,128,1,float16,float16,0,0.07794399857521057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,8,128,1,fp8,fp8,0,0.2652479887008667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,16,128,1,float16,float16,0,0.25303680896759034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,16,128,1,float16,fp8,0,0.2546639919281006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,16,128,1,fp8,fp8,0,0.2539535999298096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,1,128,1,float16,fp8,0,0.08210240006446838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,1,128,1,fp8,fp8,0,0.08133119940757752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,2,128,1,float16,float16,0,0.08685920238494874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,2,128,1,float16,fp8,0,0.08938720226287841
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,2,128,1,fp8,fp8,0,0.08884000182151794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,4,128,1,float16,float16,0,0.10719679594039917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,4,128,1,float16,fp8,0,0.10780160427093506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,4,128,1,fp8,fp8,0,0.10746400356292725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,8,128,1,float16,float16,0,0.14703359603881835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,8,128,1,float16,fp8,0,0.14330559968948364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,8,128,1,fp8,fp8,0,0.143449604511261
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,16,128,1,float16,float16,0,0.13716959953308105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,16,128,1,float16,fp8,0,0.13651520013809204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,16,128,1,fp8,fp8,0,0.1358240008354187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,1,128,1,float16,float16,0,0.04562720060348511
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,1,128,1,float16,fp8,0,0.04407199919223785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,1,128,1,fp8,fp8,0,0.04256640076637268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,2,128,1,float16,float16,0,0.05452160239219665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,2,128,1,float16,fp8,0,0.048763200640678406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,2,128,1,fp8,fp8,0,0.048316800594329835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,4,128,1,float16,float16,0,0.06385440230369568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,4,128,1,float16,fp8,0,0.057145601511001586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,4,128,1,fp8,fp8,0,0.05769280195236206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,8,128,1,float16,float16,0,0.08334239721298217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,8,128,1,float16,fp8,0,0.08103839755058288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,8,128,1,fp8,fp8,0,0.0803600013256073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,16,128,1,float16,float16,0,0.07887840270996094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,16,128,1,float16,fp8,0,0.06994720101356507
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,16,128,1,fp8,fp8,0,0.06894720196723939
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,2,128,1,fp8,fp8,0,0.03116160035133362
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,1,128,1,float16,float16,0,0.029260799288749695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,1,128,1,float16,fp8,0,0.030551999807357788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,1,128,1,fp8,fp8,0,0.030446401238441466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,2,128,1,float16,float16,0,0.029844799637794496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,2,128,1,float16,fp8,0,0.031118398904800414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,4,128,1,float16,float16,0,0.03466559946537018
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,4,128,1,float16,fp8,0,0.03572160005569458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,4,128,1,fp8,fp8,0,0.03550240099430084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,8,128,1,float16,float16,0,0.04439840018749237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,8,128,1,float16,fp8,0,0.042710399627685545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,8,128,1,fp8,fp8,0,0.04287360012531281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,16,128,1,float16,float16,0,0.04103519916534424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,16,128,1,float16,fp8,0,0.04099999964237213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,16,128,1,fp8,fp8,0,0.04095360040664673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,1,128,1,float16,float16,0,0.024113599956035615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,1,128,1,float16,fp8,0,0.024512000381946564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,1,128,1,fp8,fp8,0,0.024900799989700316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,2,128,1,float16,float16,0,0.024163199961185454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,2,128,1,float16,fp8,0,0.02483679950237274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,2,128,1,fp8,fp8,0,0.02465119957923889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,4,128,1,float16,float16,0,0.02486719936132431
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,4,128,1,float16,fp8,0,0.025588798522949218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,4,128,1,fp8,fp8,0,0.02571359872817993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,8,128,1,float16,float16,0,0.029227200150489806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,8,128,1,float16,fp8,0,0.030001598596572875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,8,128,1,fp8,fp8,0,0.030313599109649658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,16,128,1,float16,float16,0,0.028439998626708984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,16,128,1,float16,fp8,0,0.029193601012229918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,16,128,1,fp8,fp8,0,0.029207998514175416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,1,128,1,float16,float16,0,0.0235167995095253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,1,128,1,float16,fp8,0,0.02449440062046051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,1,128,1,fp8,fp8,0,0.024489599466323852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,2,128,1,float16,float16,0,0.02327679991722107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,2,128,1,float16,fp8,0,0.024139200150966645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,2,128,1,fp8,fp8,0,0.024619199335575104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,4,128,1,float16,float16,0,0.023707200586795808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,16,128,1,float16,fp8,0,0.02374880015850067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,4,128,1,float16,fp8,0,0.024187199771404266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,4,128,1,fp8,fp8,0,0.024676799774169922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,8,128,1,float16,float16,0,0.024743999540805816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,8,128,1,float16,fp8,0,0.02521440088748932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,8,128,1,fp8,fp8,0,0.025177600979804992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,16,128,1,float16,float16,0,0.02277279943227768
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,16,128,1,fp8,fp8,0,0.02373439967632294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,1,128,1,float16,float16,0,0.021822400391101837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,1,128,1,float16,fp8,0,0.022993600368499754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,1,128,1,fp8,fp8,0,0.02279839962720871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,2,128,1,float16,float16,0,0.022300800681114195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,2,128,1,float16,fp8,0,0.022737599909305573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,2,128,1,fp8,fp8,0,0.022571200132369997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,4,128,1,float16,float16,0,0.022096000611782074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,4,128,1,float16,fp8,0,0.022603200376033784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,4,128,1,fp8,fp8,0,0.02281759977340698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,8,128,1,float16,float16,0,0.022892799973487855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,8,128,1,float16,fp8,0,0.023071999847888946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,8,128,1,fp8,fp8,0,0.02290399968624115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,16,128,1,float16,float16,0,0.021222400665283202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,16,128,1,float16,fp8,0,0.0221343994140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,16,128,1,fp8,fp8,0,0.022140799462795256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,1,128,1,float16,float16,0,0.02099359929561615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,1,128,1,float16,fp8,0,0.02194560021162033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,1,128,1,fp8,fp8,0,0.022124800086021423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,2,128,1,float16,float16,0,0.02083680033683777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,2,128,1,float16,fp8,0,0.02194560021162033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,2,128,1,fp8,fp8,0,0.021635200083255767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,4,128,1,float16,float16,0,0.021086399257183076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,4,128,1,float16,fp8,0,0.022064000368118286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,4,128,1,fp8,fp8,0,0.021840000152587892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,8,128,1,float16,float16,0,0.021038399636745454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,8,128,1,float16,fp8,0,0.021768000721931458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,8,128,1,fp8,fp8,0,0.021718400716781616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,16,128,1,float16,float16,0,0.02072640061378479
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,16,128,1,float16,fp8,0,0.02136639952659607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,16,128,1,fp8,fp8,0,0.02160799950361252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,1,128,1,float16,float16,0,0.02085919976234436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,1,128,1,float16,fp8,0,0.021388800442218782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,1,128,1,fp8,fp8,0,0.021275199949741364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,2,128,1,float16,float16,0,0.020529599487781526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,2,128,1,float16,fp8,0,0.02146400064229965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,2,128,1,fp8,fp8,0,0.02146880030632019
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,4,128,1,float16,float16,0,0.020545600354671477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,4,128,1,float16,fp8,0,0.02165440022945404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,4,128,1,fp8,fp8,0,0.021622399985790252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,8,128,1,float16,float16,0,0.02082560062408447
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,8,128,1,float16,fp8,0,0.021473599970340727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,8,128,1,fp8,fp8,0,0.021401600539684297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,1,128,1,float16,float16,0,0.08009439706802368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,1,128,1,float16,fp8,0,0.08285120129585266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,1,128,1,fp8,fp8,0,0.082915198802948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,2,128,1,float16,float16,0,0.08804960250854492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,2,128,1,float16,fp8,0,0.09135519862174987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,2,128,1,fp8,fp8,0,0.09087039828300476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,4,128,1,float16,float16,0,0.10747519731521607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,4,128,1,float16,fp8,0,0.10880800485610961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,4,128,1,fp8,fp8,0,0.10861439704895019
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,8,128,1,float16,float16,0,0.17574080228805541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,8,128,1,float16,fp8,0,0.178603196144104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,8,128,1,fp8,fp8,0,0.1787935972213745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,16,128,1,float16,float16,0,0.1677999973297119
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,16,128,1,float16,fp8,0,0.17219359874725343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,16,128,1,fp8,fp8,0,0.17204320430755615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,1,128,1,float16,float16,0,0.045737600326538085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,1,128,1,float16,fp8,0,0.043532800674438474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,1,128,1,fp8,fp8,0,0.04359999895095825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,2,128,1,float16,float16,0,0.05465919971466064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,2,128,1,float16,fp8,0,0.04904319941997528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,2,128,1,fp8,fp8,0,0.04986239969730377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,4,128,1,float16,float16,0,0.06412000060081482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,4,128,1,float16,fp8,0,0.0576416015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,4,128,1,fp8,fp8,0,0.05941759943962097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,8,128,1,float16,float16,0,0.09932799935340882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,8,128,1,float16,fp8,0,0.0973695993423462
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,8,128,1,fp8,fp8,0,0.09863680005073547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,16,128,1,float16,float16,0,0.09377920031547546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,16,128,1,float16,fp8,0,0.08992159962654114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,16,128,1,fp8,fp8,0,0.08773919939994812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,1,128,1,float16,float16,0,0.029447999596595765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,1,128,1,float16,fp8,0,0.031118398904800414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,1,128,1,fp8,fp8,0,0.03091840147972107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,2,128,1,float16,float16,0,0.030302399396896364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,2,128,1,float16,fp8,0,0.03170880079269409
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,2,128,1,fp8,fp8,0,0.0318336009979248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,4,128,1,float16,float16,0,0.03590559959411621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,4,128,1,float16,fp8,0,0.035835200548172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,4,128,1,fp8,fp8,0,0.03608480095863342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,8,128,1,float16,float16,0,0.05308319926261902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,8,128,1,float16,fp8,0,0.05183359980583191
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,8,128,1,fp8,fp8,0,0.051857602596282956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,16,128,1,float16,float16,0,0.047758400440216064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,16,128,1,float16,fp8,0,0.05035039782524109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,16,128,1,fp8,fp8,0,0.050678402185440063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,1,128,1,float16,float16,0,0.024217599630355836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,1,128,1,float16,fp8,0,0.025113600492477416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,1,128,1,fp8,fp8,0,0.025183999538421632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,2,128,1,float16,float16,0,0.02483839988708496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,2,128,1,float16,fp8,0,0.02528960108757019
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,2,128,1,fp8,fp8,0,0.025271999835968017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,4,128,1,float16,float16,0,0.025067201256752013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,4,128,1,float16,fp8,0,0.026107200980186464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,4,128,1,fp8,fp8,0,0.025913599133491515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,8,128,1,float16,float16,0,0.032846400141716005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,8,128,1,float16,fp8,0,0.03436160087585449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,8,128,1,fp8,fp8,0,0.03438880145549774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,16,128,1,float16,float16,0,0.03195039927959442
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,16,128,1,float16,fp8,0,0.033851200342178346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,16,128,1,fp8,fp8,0,0.033825600147247316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,1,128,1,float16,float16,0,0.023887999355793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,1,128,1,float16,fp8,0,0.024592000246047973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,1,128,1,fp8,fp8,0,0.024297599494457246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,2,128,1,float16,float16,0,0.023294399678707122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,2,128,1,float16,fp8,0,0.02401600033044815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,2,128,1,fp8,fp8,0,0.024271999299526215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,4,128,1,float16,float16,0,0.024027200043201448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,4,128,1,float16,fp8,0,0.02452480047941208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,4,128,1,fp8,fp8,0,0.02465759962797165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,8,128,1,float16,float16,0,0.024059200286865236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,8,128,1,float16,fp8,0,0.025003200769424437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,1,128,1,float16,fp8,0,0.02263679951429367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,8,128,1,fp8,fp8,0,0.02495039999485016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,16,128,1,float16,float16,0,0.023259200155735016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,16,128,1,float16,fp8,0,0.024059200286865236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,16,128,1,fp8,fp8,0,0.02391040027141571
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,1,128,1,float16,float16,0,0.022217600047588347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,1,128,1,fp8,fp8,0,0.022793599963188173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,2,128,1,float16,float16,0,0.021891200542449953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,2,128,1,float16,fp8,0,0.02284799963235855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,2,128,1,fp8,fp8,0,0.023391999304294586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,4,128,1,float16,float16,0,0.022388799488544463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,16,128,1,float16,float16,0,0.02119999974966049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,4,128,1,float16,fp8,0,0.02332639992237091
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,4,128,1,fp8,fp8,0,0.023073600232601167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,8,128,1,float16,float16,0,0.022100800275802614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,8,128,1,float16,fp8,0,0.022860799729824067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,8,128,1,fp8,fp8,0,0.022700800001621245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,16,128,1,float16,fp8,0,0.022300800681114195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,16,128,1,fp8,fp8,0,0.02248319983482361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,1,128,1,float16,float16,0,0.02128800004720688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,1,128,1,float16,fp8,0,0.02178560048341751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,1,128,1,fp8,fp8,0,0.0220223993062973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,2,128,1,float16,float16,0,0.020950399339199066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,8,128,1,float16,fp8,0,0.02184640020132065
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,2,128,1,float16,fp8,0,0.021673600375652313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,2,128,1,fp8,fp8,0,0.02203039973974228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,4,128,1,float16,float16,0,0.02110079973936081
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,4,128,1,float16,fp8,0,0.0221791997551918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,4,128,1,fp8,fp8,0,0.02176000028848648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,8,128,1,float16,float16,0,0.021220800280570985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,8,128,1,fp8,fp8,0,0.022089600563049316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,16,128,1,float16,float16,0,0.020950399339199066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,2,128,1,float16,fp8,0,0.021296000480651854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,16,128,1,float16,fp8,0,0.021507200598716737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,16,128,1,fp8,fp8,0,0.021745599806308746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,1,128,1,float16,float16,0,0.020601600408554077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,1,128,1,float16,fp8,0,0.021491199731826782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,1,128,1,fp8,fp8,0,0.021193599700927733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,2,128,1,float16,float16,0,0.020790399610996248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,2,128,1,fp8,fp8,0,0.02141280025243759
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,4,128,1,float16,float16,0,0.020662400126457214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,4,128,1,float16,fp8,0,0.02141920030117035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,4,128,1,fp8,fp8,0,0.02151840031147003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,8,128,1,float16,float16,0,0.020707200467586517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,8,128,1,float16,fp8,0,0.02168159931898117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,8,128,1,fp8,fp8,0,0.021540799736976625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,2,128,1,float16,float16,0,0.020619200170040132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,16,128,1,float16,float16,0,0.0209184005856514
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,16,128,1,float16,fp8,0,0.02142080068588257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,16,128,1,fp8,fp8,0,0.021435199677944182
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,1,128,1,float16,float16,0,0.020431999862194062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,1,128,1,float16,fp8,0,0.021619200706481934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,1,128,1,fp8,fp8,0,0.021457600593566894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,2,128,1,float16,fp8,0,0.02126239985227585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,2,128,1,fp8,fp8,0,0.021529600024223328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,4,128,1,float16,float16,0,0.02064799964427948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,4,128,1,float16,fp8,0,0.02117439955472946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,4,128,1,fp8,fp8,0,0.021185599267482758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,8,128,1,float16,float16,0,0.02028000056743622
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,8,128,1,float16,fp8,0,0.021264000236988066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,8,128,1,fp8,fp8,0,0.02125120013952255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,1,128,1,float16,float16,0,0.04691520035266876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,1,128,1,float16,fp8,0,0.0457071989774704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,1,128,1,fp8,fp8,0,0.04651840031147003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,2,128,1,float16,float16,0,0.05628799796104431
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,2,128,1,float16,fp8,0,0.05116159915924072
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,2,128,1,fp8,fp8,0,0.05087680220603943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,4,128,1,float16,float16,0,0.08047040104866028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,4,128,1,float16,fp8,0,0.07824159860610962
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,4,128,1,fp8,fp8,0,0.07702239751815795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,8,128,1,float16,float16,0,0.1284111976623535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,8,128,1,float16,fp8,0,0.13729920387268066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,8,128,1,fp8,fp8,0,0.13652000427246094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,16,128,1,float16,float16,0,0.12484639883041382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,16,128,1,float16,fp8,0,0.12358720302581787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,16,128,1,fp8,fp8,0,0.1258847951889038
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,1,128,1,float16,float16,0,0.0302592009305954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,1,128,1,float16,fp8,0,0.031814399361610415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,1,128,1,fp8,fp8,0,0.031963199377059937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,2,128,1,float16,float16,0,0.0318015992641449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,2,128,1,float16,fp8,0,0.03299359977245331
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,2,128,1,fp8,fp8,0,0.032974401116371156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,4,128,1,float16,float16,0,0.04298399984836578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,4,128,1,float16,fp8,0,0.04592320024967193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,4,128,1,fp8,fp8,0,0.04571839869022369
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,8,128,1,float16,float16,0,0.06773279905319214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,8,128,1,float16,fp8,0,0.07073919773101807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,8,128,1,fp8,fp8,0,0.07079520225524902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,16,128,1,float16,float16,0,0.06367840170860291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,16,128,1,float16,fp8,0,0.06886240243911743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,16,128,1,fp8,fp8,0,0.06946240067481994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,1,128,1,float16,float16,0,0.02468640059232712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,1,128,1,float16,fp8,0,0.02558720111846924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,1,128,1,fp8,fp8,0,0.025593599677085875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,2,128,1,float16,float16,0,0.025255998969078063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,2,128,1,float16,fp8,0,0.025833600759506227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,2,128,1,fp8,fp8,0,0.025963199138641358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,4,128,1,float16,float16,0,0.029393601417541503
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,4,128,1,float16,fp8,0,0.030851200222969055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,4,128,1,fp8,fp8,0,0.03073279857635498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,8,128,1,float16,float16,0,0.04084480106830597
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,8,128,1,float16,fp8,0,0.043793600797653195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,8,128,1,fp8,fp8,0,0.043638399243354796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,16,128,1,float16,float16,0,0.039776000380516055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,16,128,1,float16,fp8,0,0.04344640076160431
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,16,128,1,fp8,fp8,0,0.04326240122318268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,1,128,1,float16,float16,0,0.024540799856185912
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,1,128,1,float16,fp8,0,0.025124800205230714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,1,128,1,fp8,fp8,0,0.024480000138282776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,2,128,1,float16,float16,0,0.02372799962759018
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,2,128,1,float16,fp8,0,0.024452799558639528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,2,128,1,fp8,fp8,0,0.024900799989700316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,4,128,1,float16,float16,0,0.0237744003534317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,4,128,1,float16,fp8,0,0.024689599871635437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,4,128,1,fp8,fp8,0,0.02465119957923889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,8,128,1,float16,float16,0,0.028415998816490172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,8,128,1,float16,fp8,0,0.02967199981212616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,8,128,1,fp8,fp8,0,0.029798400402069092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,16,128,1,float16,float16,0,0.02722879946231842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,16,128,1,float16,fp8,0,0.02847039997577667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,16,128,1,fp8,fp8,0,0.02829119861125946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,1,128,1,float16,float16,0,0.02213120013475418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,1,128,1,fp8,fp8,0,0.02341119945049286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,1,128,1,float16,fp8,0,0.02281759977340698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,2,128,1,float16,float16,0,0.022353599965572356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,2,128,1,float16,fp8,0,0.023183999955654143
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,2,128,1,fp8,fp8,0,0.022935999929904936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,4,128,1,float16,float16,0,0.021942399442195892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,4,128,1,float16,fp8,0,0.022756800055503845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,4,128,1,fp8,fp8,0,0.023137600719928743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,8,128,1,float16,float16,0,0.022416000068187714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,8,128,1,float16,fp8,0,0.02305919975042343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,8,128,1,fp8,fp8,0,0.022969600558280946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,16,128,1,float16,float16,0,0.021537600457668303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,16,128,1,float16,fp8,0,0.022468799352645875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,16,128,1,fp8,fp8,0,0.022457599639892578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,1,128,1,float16,float16,0,0.0211776003241539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,1,128,1,float16,fp8,0,0.021931199729442595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,1,128,1,fp8,fp8,0,0.022201600670814513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,2,128,1,float16,float16,0,0.02128159999847412
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,2,128,1,float16,fp8,0,0.02205760031938553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,2,128,1,fp8,fp8,0,0.022071999311447144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,4,128,1,float16,float16,0,0.021275199949741364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,4,128,1,float16,fp8,0,0.02205760031938553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,4,128,1,fp8,fp8,0,0.022011199593544008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,8,128,1,float16,float16,0,0.021524800360202788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,8,128,1,float16,fp8,0,0.022137600183486938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,8,128,1,fp8,fp8,0,0.021991999447345735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,16,128,1,float16,float16,0,0.02096959948539734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,16,128,1,float16,fp8,0,0.021751999855041504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,16,128,1,fp8,fp8,0,0.022148799896240235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,1,128,1,float16,float16,0,0.020695999264717102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,4,128,1,float16,fp8,0,0.02126239985227585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,1,128,1,float16,fp8,0,0.02122880071401596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,1,128,1,fp8,fp8,0,0.02157119959592819
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,2,128,1,float16,float16,0,0.020664000511169435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,2,128,1,float16,fp8,0,0.021414400637149812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,2,128,1,fp8,fp8,0,0.02144320011138916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,4,128,1,float16,float16,0,0.02062080055475235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,4,128,1,fp8,fp8,0,0.02146880030632019
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,8,128,1,float16,float16,0,0.02104319930076599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,8,128,1,float16,fp8,0,0.021716800332069398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,8,128,1,fp8,fp8,0,0.021862399578094483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,16,128,1,float16,float16,0,0.020985600352287293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,16,128,1,float16,fp8,0,0.021726399660110474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,16,128,1,fp8,fp8,0,0.02166240066289902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,1,128,1,float16,float16,0,0.0205487996339798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,1,128,1,float16,fp8,0,0.02160000056028366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,1,128,1,fp8,fp8,0,0.02144480049610138
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,2,128,1,float16,float16,0,0.020628799498081208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,2,128,1,float16,fp8,0,0.021486400067806243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,2,128,1,fp8,fp8,0,0.021359999477863312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,4,128,1,float16,float16,0,0.02051360011100769
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,4,128,1,float16,fp8,0,0.020982399582862854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,4,128,1,fp8,fp8,0,0.020996800065040587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,8,128,1,float16,float16,0,0.020615999400615693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,8,128,1,float16,fp8,0,0.02136320024728775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,8,128,1,fp8,fp8,0,0.02127680033445358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,16,128,1,float16,float16,0,0.020633600652217865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,16,128,1,float16,fp8,0,0.021433599293231964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,16,128,1,fp8,fp8,0,0.02128800004720688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,1,128,1,float16,float16,0,0.0201664000749588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,1,128,1,float16,fp8,0,0.021252800524234772
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,1,128,1,fp8,fp8,0,0.021121600270271303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,2,128,1,float16,float16,0,0.020508800446987153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,2,128,1,float16,fp8,0,0.0213919997215271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,2,128,1,fp8,fp8,0,0.021353599429130555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,4,128,1,float16,float16,0,0.020336000621318816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,4,128,1,float16,fp8,0,0.020977599918842314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,4,128,1,fp8,fp8,0,0.02120479941368103
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,8,128,1,float16,float16,0,0.020399999618530274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,8,128,1,float16,fp8,0,0.021462400257587434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,8,128,1,fp8,fp8,0,0.02123039960861206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,1,128,1,float16,fp8,0,9.53711166381836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,1,128,1,fp8,fp8,0,9.382195281982423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,2,128,1,float16,fp8,0,9.594261169433594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,2,128,1,fp8,fp8,0,9.59751205444336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,1,128,1,float16,float16,0,11.132952117919922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,2,128,1,float16,float16,0,11.385174560546876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,4,128,1,float16,float16,0,11.400287628173828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,12,128,1,float16,fp8,0,5.502891159057617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,12,128,1,fp8,fp8,0,5.502435302734375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,12,128,1,float16,float16,0,6.438974761962891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,1,128,1,float16,float16,0,5.501015853881836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,1,128,1,float16,fp8,0,4.797772979736328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,4,128,1,float16,fp8,0,9.844014739990234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,1,128,1,fp8,fp8,0,4.691791915893555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,4,128,1,fp8,fp8,0,9.885529327392579
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,12,128,1,float16,float16,0,3.143020820617676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,2,128,1,float16,fp8,0,4.790403366088867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,2,128,1,fp8,fp8,0,4.791571044921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,4,128,1,float16,fp8,0,5.012955093383789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,2,128,1,float16,float16,0,5.738817596435547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,4,128,1,fp8,fp8,0,4.924228668212891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,4,128,1,float16,float16,0,5.719900894165039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,12,128,1,float16,fp8,0,2.775974464416504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,12,128,1,fp8,fp8,0,2.8755903244018555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,1,128,1,float16,float16,0,2.6358495712280274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,1,128,1,float16,fp8,0,2.3611776351928713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,1,128,1,fp8,fp8,0,2.36474552154541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,2,128,1,float16,fp8,0,2.4068319320678713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,2,128,1,float16,float16,0,2.729257583618164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,2,128,1,fp8,fp8,0,2.4642911911010743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,12,128,1,float16,float16,0,1.541470432281494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,4,128,1,float16,float16,0,2.9003936767578127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,12,128,1,float16,fp8,0,1.4041728019714355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,4,128,1,float16,fp8,0,2.4843584060668946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,12,128,1,fp8,fp8,0,1.404315185546875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,1,128,1,float16,float16,0,1.423419189453125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,1,128,1,float16,fp8,0,1.1957407951354981
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,4,128,1,fp8,fp8,0,2.6619792938232423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,1,128,1,fp8,fp8,0,1.2936511993408204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,2,128,1,float16,fp8,0,1.2178303718566894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,2,128,1,float16,float16,0,1.4011903762817384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,2,128,1,fp8,fp8,0,1.2082415580749513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,4,128,1,float16,float16,0,1.3639087677001953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,4,128,1,fp8,fp8,0,1.25457763671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,4,128,1,float16,fp8,0,1.4474207878112793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,1,128,1,float16,fp8,0,5.389984130859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,1,128,1,fp8,fp8,0,6.010007858276367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,1,128,1,float16,float16,0,6.646177673339844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,2,128,1,float16,fp8,0,5.551876831054687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,2,128,1,fp8,fp8,0,5.623388671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,4,128,1,float16,fp8,0,5.732489776611328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,2,128,1,float16,float16,0,6.833814239501953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,4,128,1,float16,float16,0,7.6484626770019535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,1,128,1,float16,fp8,0,2.828683280944824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,1,128,1,float16,float16,0,3.1846975326538085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,12,128,1,fp8,fp8,0,3.3699230194091796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,1,128,1,fp8,fp8,0,3.0895824432373047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,12,128,1,float16,fp8,0,3.8309856414794923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,12,128,1,float16,float16,0,4.2475841522216795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,4,128,1,fp8,fp8,0,6.062823867797851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,2,128,1,float16,float16,0,3.2990432739257813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,12,128,1,float16,float16,0,1.9463951110839843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,2,128,1,float16,fp8,0,2.9289424896240233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,2,128,1,fp8,fp8,0,2.996123123168945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,4,128,1,fp8,fp8,0,2.8853567123413084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,4,128,1,float16,fp8,0,2.9561744689941407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,4,128,1,float16,float16,0,3.369174575805664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,12,128,1,float16,fp8,0,1.6933055877685548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,12,128,1,fp8,fp8,0,1.7206159591674806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,1,128,1,float16,float16,0,1.556492805480957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,1,128,1,float16,fp8,0,1.377246379852295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,1,128,1,fp8,fp8,0,1.4164591789245606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,2,128,1,float16,float16,0,1.4934800148010254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,2,128,1,float16,fp8,0,1.3977343559265136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,2,128,1,fp8,fp8,0,1.4552080154418945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,4,128,1,float16,float16,0,1.572542381286621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,12,128,1,float16,float16,0,0.9634639739990234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,4,128,1,float16,fp8,0,1.4451231956481934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,12,128,1,float16,fp8,0,0.869655990600586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,1,128,1,float16,float16,0,0.7437327861785888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,4,128,1,fp8,fp8,0,1.5787023544311523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,12,128,1,fp8,fp8,0,0.8940896034240723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,1,128,1,fp8,fp8,0,0.7481552124023437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,1,128,1,float16,fp8,0,0.6963759899139405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,2,128,1,float16,float16,0,0.7950352191925049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,2,128,1,float16,fp8,0,0.7136367797851563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,2,128,1,fp8,fp8,0,0.7133088111877441
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,4,128,1,float16,float16,0,0.8067760467529297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,4,128,1,float16,fp8,0,0.7736559867858886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,4,128,1,fp8,fp8,0,0.7823872089385986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,1,128,1,float16,fp8,0,3.785295867919922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,1,128,1,float16,float16,0,4.468652725219727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,1,128,1,fp8,fp8,0,4.003359985351563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,2,128,1,float16,fp8,0,3.9919185638427734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,2,128,1,fp8,fp8,0,3.983270263671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,2,128,1,float16,float16,0,4.674028778076172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,4,128,1,float16,float16,0,4.653449630737304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,12,128,1,float16,float16,0,2.7000783920288085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,12,128,1,float16,fp8,0,2.431865692138672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,1,128,1,float16,float16,0,2.231100845336914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,12,128,1,fp8,fp8,0,2.447358322143555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,1,128,1,float16,fp8,0,1.968017578125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,4,128,1,float16,fp8,0,4.155228805541992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,4,128,1,fp8,fp8,0,4.14721450805664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,1,128,1,fp8,fp8,0,1.9969423294067383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,2,128,1,float16,fp8,0,1.9585151672363281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,2,128,1,float16,float16,0,2.1273584365844727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,2,128,1,fp8,fp8,0,1.9873504638671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,4,128,1,float16,float16,0,2.3434223175048827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,4,128,1,float16,fp8,0,2.0577247619628904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,12,128,1,float16,float16,0,1.362497615814209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,4,128,1,fp8,fp8,0,2.052510452270508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,12,128,1,float16,fp8,0,1.2646623611450196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,1,128,1,float16,fp8,0,0.9709424018859864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,1,128,1,float16,float16,0,1.103428840637207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,12,128,1,fp8,fp8,0,1.2416463851928712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,1,128,1,fp8,fp8,0,1.092737579345703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,2,128,1,float16,float16,0,1.0562512397766113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,2,128,1,float16,fp8,0,1.0440143585205077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,2,128,1,fp8,fp8,0,0.9901375770568848
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,4,128,1,float16,float16,0,1.1180095672607422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,4,128,1,float16,fp8,0,1.0453071594238281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,12,128,1,float16,float16,0,0.7139760017395019
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,12,128,1,float16,fp8,0,0.671446418762207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,12,128,1,fp8,fp8,0,0.7029168128967285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,4,128,1,fp8,fp8,0,1.1453264236450196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,1,128,1,float16,float16,0,0.5337152004241943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,1,128,1,float16,fp8,0,0.4943088054656982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,1,128,1,fp8,fp8,0,0.5462560176849365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,2,128,1,float16,float16,0,0.5450943946838379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,2,128,1,float16,fp8,0,0.5206480026245117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,4,128,1,fp8,fp8,0,0.549777603149414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,2,128,1,fp8,fp8,0,0.522492790222168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,4,128,1,float16,float16,0,0.5684768199920655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,4,128,1,float16,fp8,0,0.550932788848877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,1,128,1,float16,fp8,0,5.010160064697265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,1,128,1,fp8,fp8,0,5.008209609985352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,1,128,1,float16,float16,0,5.89459342956543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,2,128,1,float16,fp8,0,5.174507141113281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,2,128,1,fp8,fp8,0,5.141377639770508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,2,128,1,float16,float16,0,6.028598403930664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,4,128,1,float16,fp8,0,5.4261329650878904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,4,128,1,float16,float16,0,6.4390113830566404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,1,128,1,float16,float16,0,2.982548713684082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,1,128,1,float16,fp8,0,2.508951950073242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,12,128,1,float16,float16,0,3.703713607788086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,1,128,1,fp8,fp8,0,2.4911376953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,12,128,1,fp8,fp8,0,3.274216079711914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,12,128,1,float16,fp8,0,3.4246894836425783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,2,128,1,float16,float16,0,2.9225088119506837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,4,128,1,fp8,fp8,0,5.402414321899414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,12,128,1,float16,float16,0,1.7784383773803711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,2,128,1,fp8,fp8,0,2.575649642944336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,2,128,1,float16,fp8,0,2.700328063964844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,4,128,1,float16,fp8,0,2.713827133178711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,12,128,1,float16,fp8,0,1.7584224700927735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,4,128,1,fp8,fp8,0,2.6951311111450194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,4,128,1,float16,float16,0,3.1841552734375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,12,128,1,fp8,fp8,0,1.6502511978149415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,1,128,1,float16,float16,0,1.35556640625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,1,128,1,float16,fp8,0,1.2636240005493165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,1,128,1,fp8,fp8,0,1.263923168182373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,2,128,1,float16,float16,0,1.3949567794799804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,2,128,1,float16,fp8,0,1.3388640403747558
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,2,128,1,fp8,fp8,0,1.2913904190063477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,12,128,1,float16,float16,0,0.9255359649658204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,4,128,1,float16,fp8,0,1.3639007568359376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,4,128,1,float16,float16,0,1.5752032279968262
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,4,128,1,fp8,fp8,0,1.4524352073669433
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,12,128,1,float16,fp8,0,0.8461152076721191
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,1,128,1,float16,float16,0,0.6845647811889648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,12,128,1,fp8,fp8,0,0.8638511657714844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,1,128,1,float16,fp8,0,0.6429088115692139
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,2,128,1,float16,float16,0,0.7009823799133301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,1,128,1,fp8,fp8,0,0.6598400115966797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,2,128,1,float16,fp8,0,0.6696032047271728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,2,128,1,fp8,fp8,0,0.6846752166748047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,4,128,1,float16,float16,0,0.7613056182861329
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,4,128,1,float16,fp8,0,0.7325632095336914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,12,128,1,float16,float16,0,0.4662816047668457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,4,128,1,fp8,fp8,0,0.7245903968811035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,12,128,1,float16,fp8,0,0.4381264209747314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,12,128,1,fp8,fp8,0,0.44356160163879393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,1,128,1,float16,float16,0,0.3537359952926636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,1,128,1,float16,fp8,0,0.34380159378051756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,1,128,1,fp8,fp8,0,0.34582719802856443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,4,128,1,float16,float16,0,0.37742080688476565
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,2,128,1,float16,float16,0,0.36764960289001464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,2,128,1,float16,fp8,0,0.35016961097717286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,2,128,1,fp8,fp8,0,0.3588543891906738
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,4,128,1,float16,fp8,0,0.3701440095901489
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,4,128,1,fp8,fp8,0,0.3618031978607178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,1,128,1,float16,float16,0,3.190388870239258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,1,128,1,float16,fp8,0,2.8778688430786135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,1,128,1,fp8,fp8,0,2.9470224380493164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,2,128,1,float16,fp8,0,3.0224512100219725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,2,128,1,float16,float16,0,3.44429931640625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,2,128,1,fp8,fp8,0,3.059440040588379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,4,128,1,float16,float16,0,3.6029006958007814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,12,128,1,float16,float16,0,2.2624048233032226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,1,128,1,float16,float16,0,1.5813488006591796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,12,128,1,float16,fp8,0,2.0664159774780275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,12,128,1,fp8,fp8,0,2.1906208038330077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,1,128,1,float16,fp8,0,1.5454336166381837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,4,128,1,fp8,fp8,0,3.3039295196533205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,4,128,1,float16,fp8,0,3.3679039001464846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,1,128,1,fp8,fp8,0,1.553428840637207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,2,128,1,float16,fp8,0,1.5308560371398925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,2,128,1,float16,float16,0,1.724287986755371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,2,128,1,fp8,fp8,0,1.5843839645385742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,12,128,1,float16,float16,0,1.129307174682617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,4,128,1,float16,float16,0,1.7711328506469726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,4,128,1,float16,fp8,0,1.6348976135253905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,4,128,1,fp8,fp8,0,1.6649824142456056
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,12,128,1,float16,fp8,0,1.0700511932373047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,1,128,1,float16,float16,0,0.8512495994567871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,1,128,1,float16,fp8,0,0.748360013961792
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,12,128,1,fp8,fp8,0,1.072439956665039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,1,128,1,fp8,fp8,0,0.7466703891754151
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,2,128,1,float16,float16,0,0.8290464401245117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,2,128,1,float16,fp8,0,0.7754831790924073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,2,128,1,fp8,fp8,0,0.7795487880706787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,4,128,1,float16,float16,0,0.8832688331604004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,4,128,1,float16,fp8,0,0.8639792442321778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,12,128,1,float16,float16,0,0.5990863800048828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,4,128,1,fp8,fp8,0,0.8278351783752441
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,12,128,1,float16,fp8,0,0.5500175952911377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,1,128,1,float16,float16,0,0.40102081298828124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,12,128,1,fp8,fp8,0,0.5993904113769531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,1,128,1,float16,fp8,0,0.38800160884857177
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,1,128,1,fp8,fp8,0,0.3975967884063721
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,4,128,1,float16,float16,0,0.4543471813201904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,2,128,1,float16,float16,0,0.4250607967376709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,2,128,1,float16,fp8,0,0.40970559120178224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,2,128,1,fp8,fp8,0,0.40114560127258303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,4,128,1,float16,fp8,0,0.4357327938079834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,4,128,1,fp8,fp8,0,0.445363187789917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,12,128,1,float16,float16,0,0.323140811920166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,12,128,1,float16,fp8,0,0.2869807958602905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,12,128,1,fp8,fp8,0,0.28765439987182617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,1,128,1,float16,float16,0,0.21799681186676026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,1,128,1,float16,fp8,0,0.20483839511871338
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,1,128,1,fp8,fp8,0,0.2049168109893799
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,2,128,1,float16,float16,0,0.22712318897247313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,2,128,1,float16,fp8,0,0.2152928113937378
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,2,128,1,fp8,fp8,0,0.21409120559692382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,4,128,1,float16,float16,0,0.24141919612884521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,4,128,1,float16,fp8,0,0.23199360370635985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,4,128,1,fp8,fp8,0,0.23068320751190186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,1,128,1,float16,fp8,0,2.778803253173828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,1,128,1,float16,float16,0,3.1289472579956055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,1,128,1,fp8,fp8,0,2.8016271591186523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,2,128,1,float16,float16,0,3.1927040100097654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,2,128,1,float16,fp8,0,2.9472864151000975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,2,128,1,fp8,fp8,0,2.977315139770508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,4,128,1,float16,float16,0,3.440235137939453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,4,128,1,float16,fp8,0,3.272817611694336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,1,128,1,float16,float16,0,1.5025504112243653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,1,128,1,float16,fp8,0,1.4198975563049316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,12,128,1,float16,fp8,0,2.205998420715332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,12,128,1,float16,float16,0,2.400993537902832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,1,128,1,fp8,fp8,0,1.4665087699890136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,12,128,1,fp8,fp8,0,2.2777984619140623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,2,128,1,float16,float16,0,1.569051170349121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,4,128,1,fp8,fp8,0,3.2106815338134767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,2,128,1,float16,fp8,0,1.5556159973144532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,2,128,1,fp8,fp8,0,1.4710351943969726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,4,128,1,float16,float16,0,1.7106351852416992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,12,128,1,float16,float16,0,1.1959775924682616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,12,128,1,float16,fp8,0,1.2065792083740234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,12,128,1,fp8,fp8,0,1.1100576400756836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,1,128,1,float16,float16,0,0.765388822555542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,4,128,1,fp8,fp8,0,1.616961669921875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,4,128,1,float16,fp8,0,1.6712047576904296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,1,128,1,float16,fp8,0,0.7107151985168457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,1,128,1,fp8,fp8,0,0.7541999816894531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,2,128,1,fp8,fp8,0,0.7477456092834472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,2,128,1,float16,float16,0,0.796398401260376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,2,128,1,float16,fp8,0,0.7742544174194336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,4,128,1,float16,float16,0,0.8849247932434082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,4,128,1,float16,fp8,0,0.8200192451477051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,12,128,1,float16,float16,0,0.6072783946990967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,4,128,1,fp8,fp8,0,0.8373536109924317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,12,128,1,float16,fp8,0,0.5902272224426269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,1,128,1,float16,float16,0,0.39176480770111083
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,12,128,1,fp8,fp8,0,0.5680111885070801
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,1,128,1,float16,fp8,0,0.38028481006622317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,1,128,1,fp8,fp8,0,0.3783456087112427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,2,128,1,float16,float16,0,0.40089921951293944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,2,128,1,float16,fp8,0,0.39754719734191896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,2,128,1,fp8,fp8,0,0.39782400131225587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,4,128,1,float16,float16,0,0.4474527835845947
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,4,128,1,float16,fp8,0,0.4319024085998535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,4,128,1,fp8,fp8,0,0.42467517852783204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,12,128,1,float16,float16,0,0.3203536033630371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,12,128,1,float16,fp8,0,0.30258080959320066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,12,128,1,fp8,fp8,0,0.3027856111526489
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,1,128,1,float16,float16,0,0.21024320125579835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,1,128,1,float16,fp8,0,0.20222079753875732
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,1,128,1,fp8,fp8,0,0.20285439491271973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,2,128,1,float16,float16,0,0.21492159366607666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,2,128,1,float16,fp8,0,0.21078879833221437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,2,128,1,fp8,fp8,0,0.2105423927307129
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,4,128,1,float16,float16,0,0.23841440677642822
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,4,128,1,float16,fp8,0,0.23024001121520996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,4,128,1,fp8,fp8,0,0.22897601127624512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,12,128,1,float16,float16,0,0.18995360136032105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,12,128,1,float16,fp8,0,0.17575199604034425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,12,128,1,fp8,fp8,0,0.17914400100708008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,1,128,1,float16,float16,0,0.1145632028579712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,1,128,1,float16,fp8,0,0.10868959426879883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,1,128,1,fp8,fp8,0,0.10895040035247802
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,2,128,1,float16,float16,0,0.12226400375366211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,2,128,1,float16,fp8,0,0.11404800415039062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,2,128,1,fp8,fp8,0,0.11155519485473633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,4,128,1,float16,float16,0,0.13682719469070434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,4,128,1,float16,fp8,0,0.1229375958442688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,4,128,1,fp8,fp8,0,0.12371360063552857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,1,128,1,float16,float16,0,1.7927423477172852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,1,128,1,float16,fp8,0,1.6920799255371093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,1,128,1,fp8,fp8,0,1.6959295272827148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,2,128,1,float16,float16,0,1.9008655548095703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,2,128,1,float16,fp8,0,1.7984575271606444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,2,128,1,fp8,fp8,0,1.7928335189819335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,4,128,1,float16,float16,0,2.1105440139770506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,1,128,1,float16,float16,0,0.8999551773071289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,12,128,1,float16,float16,0,1.5590208053588868
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,12,128,1,float16,fp8,0,1.469591999053955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,1,128,1,float16,fp8,0,0.848414421081543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,4,128,1,float16,fp8,0,2.1410495758056642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,4,128,1,fp8,fp8,0,2.040385627746582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,12,128,1,fp8,fp8,0,1.5530688285827636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,1,128,1,fp8,fp8,0,0.8683103561401367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,2,128,1,float16,float16,0,0.9676063537597657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,2,128,1,float16,fp8,0,0.9189696311950684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,2,128,1,fp8,fp8,0,0.9070032119750977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,4,128,1,float16,float16,0,1.0596048355102539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,4,128,1,float16,fp8,0,1.033614444732666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,12,128,1,float16,float16,0,0.7959599971771241
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,4,128,1,fp8,fp8,0,1.013811206817627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,12,128,1,fp8,fp8,0,0.8070768356323242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,12,128,1,float16,fp8,0,0.7703455924987793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,1,128,1,float16,float16,0,0.4561168193817139
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,2,128,1,float16,float16,0,0.48653759956359866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,1,128,1,float16,fp8,0,0.43810081481933594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,1,128,1,fp8,fp8,0,0.4476128101348877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,2,128,1,float16,fp8,0,0.4731344223022461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,2,128,1,fp8,fp8,0,0.4645520210266113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,4,128,1,float16,float16,0,0.55873122215271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,4,128,1,float16,fp8,0,0.5327792167663574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,12,128,1,float16,float16,0,0.41383042335510256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,4,128,1,fp8,fp8,0,0.5265999794006347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,12,128,1,float16,fp8,0,0.3968512058258057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,12,128,1,fp8,fp8,0,0.40294880867004396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,1,128,1,float16,float16,0,0.24317600727081298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,1,128,1,float16,fp8,0,0.23303039073944093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,1,128,1,fp8,fp8,0,0.235427188873291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,2,128,1,float16,float16,0,0.25347840785980225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,2,128,1,float16,fp8,0,0.250547194480896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,2,128,1,fp8,fp8,0,0.2490367889404297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,4,128,1,float16,float16,0,0.28313119411468507
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,4,128,1,float16,fp8,0,0.27810719013214114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,4,128,1,fp8,fp8,0,0.27795839309692383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,1,128,1,fp8,fp8,0,0.12173600196838379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,12,128,1,float16,float16,0,0.221612811088562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,12,128,1,float16,fp8,0,0.22035679817199708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,12,128,1,fp8,fp8,0,0.21725759506225586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,1,128,1,float16,float16,0,0.1341088056564331
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,1,128,1,float16,fp8,0,0.1250175952911377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,2,128,1,float16,float16,0,0.1429919958114624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,2,128,1,float16,fp8,0,0.13544800281524658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,2,128,1,fp8,fp8,0,0.13286240100860597
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,4,128,1,float16,float16,0,0.1560528039932251
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,4,128,1,float16,fp8,0,0.15142240524291992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,4,128,1,fp8,fp8,0,0.1490671992301941
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,12,128,1,float16,float16,0,0.1338368058204651
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,12,128,1,float16,fp8,0,0.12457760572433471
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,12,128,1,fp8,fp8,0,0.12477600574493408
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,1,128,1,float16,float16,0,0.07553279995918274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,1,128,1,float16,fp8,0,0.07130560278892517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,1,128,1,fp8,fp8,0,0.07236319780349731
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,2,128,1,float16,float16,0,0.07809119820594787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,2,128,1,float16,fp8,0,0.07540640234947205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,2,128,1,fp8,fp8,0,0.0759663999080658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,4,128,1,float16,float16,0,0.0866544008255005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,4,128,1,float16,fp8,0,0.08318719863891602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,4,128,1,fp8,fp8,0,0.08304479718208313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,1,128,1,float16,float16,0,1.757792091369629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,1,128,1,float16,fp8,0,1.6773359298706054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,1,128,1,fp8,fp8,0,1.691756820678711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,2,128,1,float16,float16,0,1.929412841796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,2,128,1,float16,fp8,0,1.8468128204345704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,2,128,1,fp8,fp8,0,1.844046401977539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,4,128,1,float16,float16,0,2.233907127380371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,12,128,1,float16,float16,0,1.7606351852416993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,4,128,1,float16,fp8,0,2.1666927337646484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,4,128,1,fp8,fp8,0,2.121993637084961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,1,128,1,float16,float16,0,0.8796064376831054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,12,128,1,float16,fp8,0,1.7010223388671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,1,128,1,float16,fp8,0,0.8975472450256348
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,12,128,1,fp8,fp8,0,1.682464027404785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,1,128,1,fp8,fp8,0,0.8453455924987793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,2,128,1,float16,float16,0,0.970359992980957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,2,128,1,float16,fp8,0,0.9202032089233398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,2,128,1,fp8,fp8,0,0.922383975982666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,4,128,1,float16,float16,0,1.1156160354614257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,4,128,1,float16,fp8,0,1.1141823768615722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,4,128,1,fp8,fp8,0,1.0689311981201173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,1,128,1,float16,float16,0,0.450867223739624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,12,128,1,float16,fp8,0,0.8688655853271484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,12,128,1,float16,float16,0,0.9019359588623047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,12,128,1,fp8,fp8,0,0.8480367660522461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,1,128,1,float16,fp8,0,0.44381279945373536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,2,128,1,float16,float16,0,0.4882512092590332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,1,128,1,fp8,fp8,0,0.4452816009521484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,2,128,1,float16,fp8,0,0.47589759826660155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,2,128,1,fp8,fp8,0,0.48227519989013673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,4,128,1,float16,float16,0,0.5763232231140136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,4,128,1,float16,fp8,0,0.5442463874816894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,4,128,1,fp8,fp8,0,0.557148790359497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,12,128,1,float16,float16,0,0.46225600242614745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,1,128,1,float16,float16,0,0.23973441123962402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,12,128,1,float16,fp8,0,0.44566879272460935
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,1,128,1,float16,fp8,0,0.2350912094116211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,2,128,1,fp8,fp8,0,0.2537343978881836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,4,128,1,float16,float16,0,0.29956960678100586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,12,128,1,fp8,fp8,0,0.45040159225463866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,1,128,1,fp8,fp8,0,0.2340847969055176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,2,128,1,float16,float16,0,0.26006720066070554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,2,128,1,float16,fp8,0,0.2539583921432495
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,4,128,1,float16,fp8,0,0.28631200790405276
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,4,128,1,fp8,fp8,0,0.29115679264068606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,12,128,1,float16,float16,0,0.2546207904815674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,12,128,1,float16,fp8,0,0.24431359767913818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,12,128,1,fp8,fp8,0,0.24309759140014647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,1,128,1,float16,float16,0,0.1341920018196106
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,4,128,1,float16,float16,0,0.1616752028465271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,1,128,1,float16,fp8,0,0.1294800043106079
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,1,128,1,fp8,fp8,0,0.1317952036857605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,2,128,1,float16,float16,0,0.14044159650802612
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,2,128,1,float16,fp8,0,0.1403823971748352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,12,128,1,fp8,fp8,0,0.1361791968345642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,2,128,1,fp8,fp8,0,0.13973599672317505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,4,128,1,float16,fp8,0,0.15783679485321045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,4,128,1,fp8,fp8,0,0.158297598361969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,12,128,1,float16,float16,0,0.1415488004684448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,12,128,1,float16,fp8,0,0.13559039831161498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,1,128,1,float16,float16,0,0.07325919866561889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,1,128,1,float16,fp8,0,0.07085599899291992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,1,128,1,fp8,fp8,0,0.07090880274772644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,2,128,1,float16,float16,0,0.08013920187950134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,2,128,1,float16,fp8,0,0.07523679733276367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,2,128,1,fp8,fp8,0,0.07509279847145081
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,4,128,1,float16,float16,0,0.09219359755516052
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,4,128,1,float16,fp8,0,0.08348159790039063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,4,128,1,fp8,fp8,0,0.08456799983978272
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,12,128,1,float16,float16,0,0.07802240252494812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,12,128,1,float16,fp8,0,0.07004479765892029
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,12,128,1,fp8,fp8,0,0.07077119946479797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,1,128,1,float16,float16,0,0.04778720140457153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,1,128,1,float16,fp8,0,0.04715040028095245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,1,128,1,fp8,fp8,0,0.047391998767852786
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,2,128,1,float16,float16,0,0.048984000086784364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,2,128,1,float16,fp8,0,0.048281601071357726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,2,128,1,fp8,fp8,0,0.04819360077381134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,4,128,1,float16,float16,0,0.05241119861602783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,4,128,1,float16,fp8,0,0.05245440006256104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,4,128,1,fp8,fp8,0,0.052305597066879275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,1,128,1,float16,float16,0,1.0825152397155762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,1,128,1,float16,fp8,0,1.062350368499756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,1,128,1,fp8,fp8,0,1.0594752311706543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,2,128,1,float16,float16,0,1.1985024452209472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,2,128,1,float16,fp8,0,1.1793343544006347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,2,128,1,fp8,fp8,0,1.17216796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,4,128,1,float16,float16,0,1.4334832191467286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,4,128,1,float16,fp8,0,1.406721591949463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,4,128,1,fp8,fp8,0,1.4090096473693847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,12,128,1,float16,float16,0,1.2051712036132813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,1,128,1,float16,float16,0,0.5478943824768067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,12,128,1,float16,fp8,0,1.1552255630493165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,2,128,1,float16,float16,0,0.6040272235870361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,1,128,1,float16,fp8,0,0.5407264232635498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,1,128,1,fp8,fp8,0,0.5447375774383545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,12,128,1,fp8,fp8,0,1.1632495880126954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,4,128,1,float16,float16,0,0.7235343933105469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,2,128,1,float16,fp8,0,0.604091215133667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,2,128,1,fp8,fp8,0,0.6035168170928955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,4,128,1,float16,fp8,0,0.7109888076782227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,4,128,1,fp8,fp8,0,0.7104959964752198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,12,128,1,float16,float16,0,0.6151360034942627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,1,128,1,float16,float16,0,0.2873759984970093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,12,128,1,float16,fp8,0,0.593836784362793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,2,128,1,float16,float16,0,0.31636478900909426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,2,128,1,float16,fp8,0,0.31177759170532227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,12,128,1,fp8,fp8,0,0.5980720043182373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,1,128,1,float16,fp8,0,0.2862911939620972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,1,128,1,fp8,fp8,0,0.2824687957763672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,2,128,1,fp8,fp8,0,0.3141200065612793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,4,128,1,float16,float16,0,0.3792783975601196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,4,128,1,float16,fp8,0,0.36675200462341306
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,4,128,1,fp8,fp8,0,0.3674815893173218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,12,128,1,float16,float16,0,0.3282831907272339
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,12,128,1,float16,fp8,0,0.30916481018066405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,12,128,1,fp8,fp8,0,0.3046000003814697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,1,128,1,float16,float16,0,0.15719519853591918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,1,128,1,float16,fp8,0,0.15543999671936035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,1,128,1,fp8,fp8,0,0.15665119886398315
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,2,128,1,float16,float16,0,0.17030400037765503
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,2,128,1,float16,fp8,0,0.1709439992904663
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,2,128,1,fp8,fp8,0,0.16877119541168212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,4,128,1,float16,float16,0,0.19947999715805054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,4,128,1,float16,fp8,0,0.19747519493103027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,4,128,1,fp8,fp8,0,0.19844800233840942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,1,128,1,float16,fp8,0,0.08268479704856872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,12,128,1,float16,float16,0,0.1779247999191284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,12,128,1,float16,fp8,0,0.16642240285873414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,12,128,1,fp8,fp8,0,0.16686079502105713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,1,128,1,float16,float16,0,0.08940960168838501
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,1,128,1,fp8,fp8,0,0.08425599932670594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,2,128,1,float16,float16,0,0.09740319848060608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,2,128,1,float16,fp8,0,0.09245120286941529
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,2,128,1,fp8,fp8,0,0.09323999881744385
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,4,128,1,float16,float16,0,0.11188160181045533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,4,128,1,float16,fp8,0,0.1094480037689209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,4,128,1,fp8,fp8,0,0.10891040563583373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,12,128,1,float16,float16,0,0.10407999753952027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,12,128,1,float16,fp8,0,0.09082559943199157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,12,128,1,fp8,fp8,0,0.08893920183181762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,1,128,1,float16,float16,0,0.051857602596282956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,1,128,1,float16,fp8,0,0.051823997497558595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,1,128,1,fp8,fp8,0,0.052121597528457644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,2,128,1,float16,float16,0,0.05493280291557312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,2,128,1,float16,fp8,0,0.053673601150512694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,2,128,1,fp8,fp8,0,0.053921598196029666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,4,128,1,float16,float16,0,0.06384000182151794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,4,128,1,float16,fp8,0,0.060387200117111205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,4,128,1,fp8,fp8,0,0.060127997398376466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,12,128,1,float16,float16,0,0.057894402742385866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,12,128,1,float16,fp8,0,0.05611680150032043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,12,128,1,fp8,fp8,0,0.05580639839172363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,1,128,1,float16,float16,0,0.04099360108375549
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,1,128,1,float16,fp8,0,0.041115200519561766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,1,128,1,fp8,fp8,0,0.04103359878063202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,2,128,1,float16,float16,0,0.04153279960155487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,2,128,1,float16,fp8,0,0.04174239933490753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,2,128,1,fp8,fp8,0,0.04239839911460876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,4,128,1,float16,float16,0,0.04415520131587982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,4,128,1,float16,fp8,0,0.04420160055160523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,4,128,1,fp8,fp8,0,0.044409599900245664
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,1,128,1,float16,float16,0,1.292574405670166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,1,128,1,float16,fp8,0,1.268951988220215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,1,128,1,fp8,fp8,0,1.2687904357910156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,2,128,1,float16,float16,0,1.444095993041992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,2,128,1,float16,fp8,0,1.436854362487793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,2,128,1,fp8,fp8,0,1.4372912406921388
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,4,128,1,float16,float16,0,1.756399917602539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,4,128,1,float16,fp8,0,1.7499391555786132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,1,128,1,float16,float16,0,0.6623040199279785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,1,128,1,float16,fp8,0,0.6489888191223144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,12,128,1,float16,float16,0,1.521326446533203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,12,128,1,float16,fp8,0,1.4767439842224122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,4,128,1,fp8,fp8,0,1.7489856719970702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,1,128,1,fp8,fp8,0,0.6517343997955323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,2,128,1,float16,float16,0,0.7426176071166992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,12,128,1,fp8,fp8,0,1.4697888374328614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,2,128,1,float16,fp8,0,0.7265552043914795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,2,128,1,fp8,fp8,0,0.7262447834014892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,4,128,1,float16,float16,0,0.8964351654052735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,4,128,1,float16,fp8,0,0.877723217010498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,4,128,1,fp8,fp8,0,0.8916432380676269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,12,128,1,float16,float16,0,0.7751071929931641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,1,128,1,float16,float16,0,0.3417952060699463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,12,128,1,float16,fp8,0,0.7457359790802002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,1,128,1,float16,fp8,0,0.3401968002319336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,12,128,1,fp8,fp8,0,0.7488416194915771
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,1,128,1,fp8,fp8,0,0.3433248043060303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,2,128,1,float16,float16,0,0.37743520736694336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,2,128,1,float16,fp8,0,0.38083999156951903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,2,128,1,fp8,fp8,0,0.378436803817749
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,4,128,1,float16,float16,0,0.45722241401672364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,4,128,1,float16,fp8,0,0.45071840286254883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,4,128,1,fp8,fp8,0,0.45772638320922854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,12,128,1,float16,float16,0,0.40073118209838865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,12,128,1,float16,fp8,0,0.392411208152771
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,1,128,1,float16,float16,0,0.1857200026512146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,12,128,1,fp8,fp8,0,0.3874527931213379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,1,128,1,float16,fp8,0,0.18697439432144164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,1,128,1,fp8,fp8,0,0.1852895975112915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,2,128,1,float16,float16,0,0.20351359844207764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,2,128,1,float16,fp8,0,0.20560319423675538
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,2,128,1,fp8,fp8,0,0.20348479747772216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,4,128,1,float16,float16,0,0.245796799659729
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,4,128,1,float16,fp8,0,0.2417776107788086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,1,128,1,float16,float16,0,0.1057695984840393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,4,128,1,fp8,fp8,0,0.24396159648895263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,12,128,1,float16,float16,0,0.21568000316619873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,12,128,1,float16,fp8,0,0.20830399990081788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,12,128,1,fp8,fp8,0,0.2095168113708496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,1,128,1,float16,fp8,0,0.10833439826965333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,1,128,1,fp8,fp8,0,0.10731040239334107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,2,128,1,float16,float16,0,0.11401760578155518
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,2,128,1,float16,fp8,0,0.11703039407730102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,2,128,1,fp8,fp8,0,0.11700960397720336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,4,128,1,float16,float16,0,0.13554240465164186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,4,128,1,float16,fp8,0,0.13719680309295654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,4,128,1,fp8,fp8,0,0.13692480325698853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,12,128,1,float16,float16,0,0.11923999786376953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,12,128,1,float16,fp8,0,0.1159551978111267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,2,128,1,float16,float16,0,0.0659280002117157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,12,128,1,fp8,fp8,0,0.11751199960708618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,2,128,1,fp8,fp8,0,0.06348800063133239
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,1,128,1,float16,float16,0,0.05959360003471374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,4,128,1,float16,fp8,0,0.07401279807090759
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,1,128,1,float16,fp8,0,0.05873280167579651
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,1,128,1,fp8,fp8,0,0.059867197275161745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,2,128,1,float16,fp8,0,0.06374880075454711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,4,128,1,float16,float16,0,0.0784608006477356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,4,128,1,fp8,fp8,0,0.07198879718780518
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,12,128,1,float16,float16,0,0.06681119799613952
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,12,128,1,float16,fp8,0,0.0626031994819641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,12,128,1,fp8,fp8,0,0.062110400199890135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,1,128,1,float16,float16,0,0.03893440067768097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,1,128,1,float16,fp8,0,0.03949280083179474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,1,128,1,fp8,fp8,0,0.039638400077819824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,2,128,1,float16,float16,0,0.03962079882621765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,2,128,1,float16,fp8,0,0.042151999473571775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,2,128,1,fp8,fp8,0,0.042630401253700254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,4,128,1,float16,float16,0,0.04414399862289429
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,4,128,1,float16,fp8,0,0.04678399860858917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,4,128,1,fp8,fp8,0,0.04666399955749512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,12,128,1,float16,float16,0,0.058487999439239505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,12,128,1,float16,fp8,0,0.057334399223327635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,12,128,1,fp8,fp8,0,0.05886080265045166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,1,128,1,float16,float16,0,0.04163999855518341
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,1,128,1,float16,fp8,0,0.04295839965343475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,1,128,1,fp8,fp8,0,0.04312160015106201
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,2,128,1,float16,float16,0,0.045105600357055665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,2,128,1,float16,fp8,0,0.0453247994184494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,2,128,1,fp8,fp8,0,0.04544320106506348
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,4,128,1,float16,float16,0,0.04747680127620697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,4,128,1,float16,fp8,0,0.04784319996833801
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,4,128,1,fp8,fp8,0,0.04781759977340698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,1,128,1,float16,float16,0,0.8533647537231446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,1,128,1,float16,fp8,0,0.8645183563232421
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,1,128,1,fp8,fp8,0,0.8641743659973145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,2,128,1,float16,float16,0,1.0050304412841797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,2,128,1,float16,fp8,0,1.0222304344177247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,2,128,1,fp8,fp8,0,1.019603157043457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,4,128,1,float16,float16,0,1.3158224105834961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,1,128,1,float16,float16,0,0.4397600173950195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,4,128,1,float16,fp8,0,1.3305423736572266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,1,128,1,float16,fp8,0,0.4447936058044434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,4,128,1,fp8,fp8,0,1.3218159675598145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,12,128,1,float16,float16,0,1.2980575561523438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,12,128,1,float16,fp8,0,1.2731439590454101
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,12,128,1,fp8,fp8,0,1.2724384307861327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,1,128,1,fp8,fp8,0,0.4425312042236328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,2,128,1,float16,float16,0,0.5177887916564942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,2,128,1,float16,fp8,0,0.5237472057342529
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,2,128,1,fp8,fp8,0,0.5210608005523681
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,4,128,1,float16,float16,0,0.6700960159301758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,4,128,1,float16,fp8,0,0.6722383975982666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,4,128,1,fp8,fp8,0,0.6735184192657471
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,12,128,1,float16,float16,0,0.6636384010314942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,1,128,1,float16,float16,0,0.23134400844573974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,12,128,1,float16,fp8,0,0.6492303848266602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,1,128,1,float16,fp8,0,0.23462400436401368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,12,128,1,fp8,fp8,0,0.6514287948608398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,1,128,1,fp8,fp8,0,0.23527200222015382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,2,128,1,float16,float16,0,0.2704495906829834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,2,128,1,float16,fp8,0,0.27245440483093264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,2,128,1,fp8,fp8,0,0.2763040065765381
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,4,128,1,float16,float16,0,0.34697599411010743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,4,128,1,float16,fp8,0,0.3485152006149292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,4,128,1,fp8,fp8,0,0.3468463897705078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,12,128,1,float16,float16,0,0.34698240756988524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,12,128,1,float16,fp8,0,0.33909759521484373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,1,128,1,float16,float16,0,0.12767839431762695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,12,128,1,fp8,fp8,0,0.33815679550170896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,1,128,1,float16,fp8,0,0.13193440437316895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,1,128,1,fp8,fp8,0,0.13043359518051148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,2,128,1,float16,float16,0,0.14810080528259278
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,2,128,1,float16,fp8,0,0.14907679557800294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,2,128,1,fp8,fp8,0,0.15056159496307372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,4,128,1,float16,float16,0,0.18801920413970946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,4,128,1,float16,fp8,0,0.18657759428024293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,4,128,1,fp8,fp8,0,0.18851360082626342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,12,128,1,float16,float16,0,0.18694560527801513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,12,128,1,float16,fp8,0,0.17960480451583863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,12,128,1,fp8,fp8,0,0.18044960498809814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,1,128,1,float16,float16,0,0.07743200063705444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,1,128,1,float16,fp8,0,0.07906559705734253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,1,128,1,fp8,fp8,0,0.07934560179710388
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,2,128,1,float16,float16,0,0.08631680011749268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,2,128,1,float16,fp8,0,0.08739519715309144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,2,128,1,fp8,fp8,0,0.08731039762496948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,4,128,1,float16,float16,0,0.10535520315170288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,4,128,1,float16,fp8,0,0.10578399896621704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,4,128,1,fp8,fp8,0,0.1069551944732666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,12,128,1,float16,float16,0,0.10532640218734741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,12,128,1,float16,fp8,0,0.10123039484024048
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,1,128,1,float16,float16,0,0.04457120001316071
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,2,128,1,float16,fp8,0,0.05128160119056702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,12,128,1,fp8,fp8,0,0.10003199577331542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,1,128,1,float16,fp8,0,0.04577600061893463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,1,128,1,fp8,fp8,0,0.045840001106262206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,2,128,1,float16,float16,0,0.05121279954910278
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,2,128,1,fp8,fp8,0,0.05100319981575012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,4,128,1,float16,float16,0,0.06360480189323425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,4,128,1,float16,fp8,0,0.05974559783935547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,4,128,1,fp8,fp8,0,0.05920959711074829
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,12,128,1,float16,float16,0,0.06029760241508484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,12,128,1,float16,fp8,0,0.055281597375869754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,12,128,1,fp8,fp8,0,0.05548319816589355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,1,128,1,float16,float16,0,0.03308640122413635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,1,128,1,float16,fp8,0,0.034790399670600894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,1,128,1,fp8,fp8,0,0.035420799255371095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,2,128,1,float16,float16,0,0.03383359909057617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,2,128,1,float16,fp8,0,0.03577120006084442
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,2,128,1,fp8,fp8,0,0.03572799861431122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,4,128,1,float16,float16,0,0.03856160044670105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,4,128,1,float16,fp8,0,0.04046719968318939
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,4,128,1,fp8,fp8,0,0.04028159976005554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,12,128,1,float16,float16,0,0.03775199949741363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,12,128,1,float16,fp8,0,0.03914879858493805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,12,128,1,fp8,fp8,0,0.03909119963645935
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,1,128,1,float16,float16,0,0.028003200888633728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,1,128,1,float16,fp8,0,0.0296640008687973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,1,128,1,fp8,fp8,0,0.029865598678588866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,2,128,1,float16,float16,0,0.028531199693679808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,2,128,1,float16,fp8,0,0.030377599596977233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,2,128,1,fp8,fp8,0,0.029791998863220214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,4,128,1,float16,float16,0,0.029548799991607665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,4,128,1,float16,fp8,0,0.030955201387405394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,4,128,1,fp8,fp8,0,0.030686399340629576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,12,128,1,float16,float16,0,0.031142398715019226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,12,128,1,float16,fp8,0,0.03284479975700379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,12,128,1,fp8,fp8,0,0.03265439867973328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,1,128,1,float16,float16,0,0.027937600016593934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,1,128,1,float16,fp8,0,0.029443201422691346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,1,128,1,fp8,fp8,0,0.029526400566101074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,2,128,1,float16,float16,0,0.02773439884185791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,2,128,1,float16,fp8,0,0.029342401027679443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,2,128,1,fp8,fp8,0,0.029275199770927428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,4,128,1,float16,float16,0,0.02823840081691742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,4,128,1,float16,fp8,0,0.029417601227760316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,4,128,1,fp8,fp8,0,0.030276799201965333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,1,128,1,float16,float16,0,0.3715552091598511
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,1,128,1,float16,fp8,0,0.38420801162719725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,1,128,1,fp8,fp8,0,0.38135519027709963
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,2,128,1,float16,float16,0,0.4497727870941162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,4,128,1,float16,float16,0,0.6072624206542969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,2,128,1,float16,fp8,0,0.46106557846069335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,2,128,1,fp8,fp8,0,0.46145119667053225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,4,128,1,fp8,fp8,0,0.6128928184509277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,4,128,1,float16,fp8,0,0.6175087928771973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,1,128,1,float16,float16,0,0.1988319993019104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,12,128,1,float16,float16,0,0.6420256137847901
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,12,128,1,float16,fp8,0,0.6055727958679199
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,1,128,1,float16,fp8,0,0.2052527904510498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,1,128,1,fp8,fp8,0,0.20597760677337645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,12,128,1,fp8,fp8,0,0.6085216045379639
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,2,128,1,float16,float16,0,0.238044810295105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,2,128,1,float16,fp8,0,0.24285759925842285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,2,128,1,fp8,fp8,0,0.24203040599822997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,4,128,1,float16,float16,0,0.3177472114562988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,1,128,1,float16,float16,0,0.11232960224151611
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,4,128,1,float16,fp8,0,0.3167903900146484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,4,128,1,fp8,fp8,0,0.317140793800354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,12,128,1,float16,float16,0,0.33276000022888186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,12,128,1,float16,fp8,0,0.3160975933074951
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,2,128,1,fp8,fp8,0,0.13382879495620728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,12,128,1,fp8,fp8,0,0.315449595451355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,4,128,1,float16,float16,0,0.17083200216293334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,1,128,1,float16,fp8,0,0.1149135947227478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,1,128,1,fp8,fp8,0,0.1157088041305542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,2,128,1,float16,float16,0,0.13156160116195678
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,2,128,1,float16,fp8,0,0.13372160196304322
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,4,128,1,float16,fp8,0,0.17167840003967286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,1,128,1,float16,fp8,0,0.07073760032653809
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,4,128,1,fp8,fp8,0,0.1708176016807556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,12,128,1,float16,float16,0,0.17934080362319946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,12,128,1,float16,fp8,0,0.16969120502471924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,2,128,1,fp8,fp8,0,0.07878239750862122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,12,128,1,fp8,fp8,0,0.16863199472427368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,1,128,1,float16,float16,0,0.06942560076713562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,1,128,1,fp8,fp8,0,0.07017599940299987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,2,128,1,float16,float16,0,0.07853599786758422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,2,128,1,float16,fp8,0,0.07926719784736633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,4,128,1,float16,float16,0,0.09802560210227966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,4,128,1,float16,fp8,0,0.09719359874725342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,4,128,1,fp8,fp8,0,0.09796640276908875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,12,128,1,float16,float16,0,0.10011199712753296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,12,128,1,float16,fp8,0,0.09550079703330994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,12,128,1,fp8,fp8,0,0.09553120136260987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,1,128,1,float16,float16,0,0.03924799859523773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,1,128,1,float16,fp8,0,0.03989599943161011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,1,128,1,fp8,fp8,0,0.039340800046920775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,2,128,1,float16,float16,0,0.04729120135307312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,2,128,1,float16,fp8,0,0.04437279999256134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,2,128,1,fp8,fp8,0,0.04435999989509583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,4,128,1,float16,float16,0,0.05886560082435608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,4,128,1,float16,fp8,0,0.053420799970626834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,4,128,1,fp8,fp8,0,0.0544048011302948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,12,128,1,float16,float16,0,0.05737919807434082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,12,128,1,float16,fp8,0,0.052108800411224364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,12,128,1,fp8,fp8,0,0.051641601324081424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,1,128,1,float16,float16,0,0.029974400997161865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,1,128,1,float16,fp8,0,0.031379199028015135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,1,128,1,fp8,fp8,0,0.031737598776817325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,2,128,1,float16,float16,0,0.030563199520111085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,2,128,1,float16,fp8,0,0.0324864000082016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,2,128,1,fp8,fp8,0,0.03228960037231445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,4,128,1,float16,float16,0,0.035150399804115294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,4,128,1,float16,fp8,0,0.0368800014257431
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,4,128,1,fp8,fp8,0,0.03671039938926697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,12,128,1,float16,float16,0,0.03438400030136109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,12,128,1,float16,fp8,0,0.03542880117893219
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,12,128,1,fp8,fp8,0,0.03516159951686859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,1,128,1,float16,float16,0,0.025076800584793092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,1,128,1,float16,fp8,0,0.026043200492858888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,1,128,1,fp8,fp8,0,0.02600319981575012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,2,128,1,float16,float16,0,0.025417599081993102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,2,128,1,float16,fp8,0,0.02640160024166107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,2,128,1,fp8,fp8,0,0.026475200057029726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,4,128,1,float16,float16,0,0.02613919973373413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,4,128,1,float16,fp8,0,0.027215999364852906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,4,128,1,fp8,fp8,0,0.027060800790786745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,12,128,1,float16,float16,0,0.02800639867782593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,12,128,1,float16,fp8,0,0.02905600070953369
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,12,128,1,fp8,fp8,0,0.029084798693656922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,1,128,1,float16,float16,0,0.02492479979991913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,1,128,1,float16,fp8,0,0.02582559883594513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,1,128,1,fp8,fp8,0,0.025920000672340394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,2,128,1,float16,float16,0,0.024721600115299225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,2,128,1,float16,fp8,0,0.025908800959587096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,2,128,1,fp8,fp8,0,0.025835201144218445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,4,128,1,float16,float16,0,0.02499520033597946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,4,128,1,float16,fp8,0,0.026020801067352294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,4,128,1,fp8,fp8,0,0.026505601406097413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,12,128,1,float16,float16,0,0.024190400540828706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,12,128,1,float16,fp8,0,0.025363200902938844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,12,128,1,fp8,fp8,0,0.025316798686981203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,1,128,1,float16,float16,0,0.023497599363327026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,1,128,1,float16,fp8,0,0.024465599656105043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,1,128,1,fp8,fp8,0,0.02444639950990677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,2,128,1,float16,float16,0,0.023694400489330292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,2,128,1,float16,fp8,0,0.02476159930229187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,2,128,1,fp8,fp8,0,0.02449920028448105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,1,128,1,float16,fp8,0,0.20187199115753174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,4,128,1,float16,float16,0,0.02382880002260208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,4,128,1,float16,fp8,0,0.02444159984588623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,4,128,1,fp8,fp8,0,0.024825599789619446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,1,128,1,float16,float16,0,0.19250400066375734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,1,128,1,fp8,fp8,0,0.20472960472106932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,2,128,1,float16,float16,0,0.22958080768585204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,2,128,1,float16,fp8,0,0.24496800899505616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,2,128,1,fp8,fp8,0,0.24348959922790528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,12,128,1,float16,float16,0,0.33552160263061526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,4,128,1,float16,float16,0,0.3147216081619263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,4,128,1,float16,fp8,0,0.32070560455322267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,4,128,1,fp8,fp8,0,0.32093760967254636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,12,128,1,float16,fp8,0,0.3189856052398682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,1,128,1,float16,float16,0,0.10863200426101685
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,12,128,1,fp8,fp8,0,0.31780960559844973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,1,128,1,float16,fp8,0,0.11327199935913086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,1,128,1,fp8,fp8,0,0.11377120018005371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,2,128,1,float16,float16,0,0.1285055994987488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,2,128,1,float16,fp8,0,0.13220479488372802
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,2,128,1,fp8,fp8,0,0.13176000118255615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,4,128,1,float16,float16,0,0.17057119607925414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,4,128,1,float16,fp8,0,0.17093600034713746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,4,128,1,fp8,fp8,0,0.17072000503540039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,12,128,1,float16,float16,0,0.18007680177688598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,12,128,1,float16,fp8,0,0.16861599683761597
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,12,128,1,fp8,fp8,0,0.1683568000793457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,1,128,1,float16,float16,0,0.06715520024299622
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,1,128,1,float16,fp8,0,0.06779360175132751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,1,128,1,fp8,fp8,0,0.06778079867362977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,2,128,1,float16,float16,0,0.07667999863624572
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,2,128,1,float16,fp8,0,0.07669280171394348
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,12,128,1,float16,fp8,0,0.09185919761657715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,2,128,1,fp8,fp8,0,0.07582240104675293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,4,128,1,float16,float16,0,0.09645119905471802
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,4,128,1,float16,fp8,0,0.09503359794616699
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,4,128,1,fp8,fp8,0,0.09471359848976135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,12,128,1,float16,float16,0,0.09858239889144897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,12,128,1,fp8,fp8,0,0.09176959991455078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,1,128,1,float16,float16,0,0.03604640066623688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,1,128,1,float16,fp8,0,0.036494401097297666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,1,128,1,fp8,fp8,0,0.03664000034332275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,2,128,1,float16,float16,0,0.04346239864826203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,2,128,1,float16,fp8,0,0.04115839898586273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,2,128,1,fp8,fp8,0,0.041652798652648926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,4,128,1,float16,float16,0,0.057233601808547974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,4,128,1,float16,fp8,0,0.0495855987071991
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,4,128,1,fp8,fp8,0,0.04905759990215301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,12,128,1,float16,float16,0,0.057126402854919434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,12,128,1,float16,fp8,0,0.04995039999485016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,12,128,1,fp8,fp8,0,0.04984639883041382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,1,128,1,float16,float16,0,0.028388801217079162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,1,128,1,float16,fp8,0,0.029817599058151244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,1,128,1,fp8,fp8,0,0.0298880010843277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,2,128,1,float16,float16,0,0.02929440140724182
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,2,128,1,float16,fp8,0,0.03060159981250763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,2,128,1,fp8,fp8,0,0.030671998858451843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,4,128,1,float16,float16,0,0.0339711993932724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,4,128,1,float16,fp8,0,0.03464959859848023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,4,128,1,fp8,fp8,0,0.03494719862937927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,12,128,1,float16,float16,0,0.03315359950065613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,12,128,1,float16,fp8,0,0.03337279856204987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,12,128,1,fp8,fp8,0,0.033353599905967715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,1,128,1,float16,float16,0,0.02370080053806305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,1,128,1,float16,fp8,0,0.02470560073852539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,1,128,1,fp8,fp8,0,0.024132800102233887
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,2,128,1,float16,float16,0,0.024289600551128387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,2,128,1,float16,fp8,0,0.024772800505161285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,2,128,1,fp8,fp8,0,0.024822400510311128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,12,128,1,float16,fp8,0,0.027134400606155396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,4,128,1,float16,float16,0,0.024864000082015992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,4,128,1,float16,fp8,0,0.025571200251579284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,4,128,1,fp8,fp8,0,0.0254831999540329
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,12,128,1,float16,float16,0,0.026649600267410277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,12,128,1,fp8,fp8,0,0.02731359899044037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,1,128,1,float16,float16,0,0.0234047994017601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,1,128,1,float16,fp8,0,0.02412640005350113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,1,128,1,fp8,fp8,0,0.02423200011253357
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,2,128,1,float16,float16,0,0.023736000061035156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,2,128,1,float16,fp8,0,0.024040000140666963
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,2,128,1,fp8,fp8,0,0.02409760057926178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,4,128,1,float16,float16,0,0.023841600120067596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,4,128,1,float16,fp8,0,0.024244800209999084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,4,128,1,fp8,fp8,0,0.024465599656105043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,12,128,1,float16,float16,0,0.02276960015296936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,12,128,1,float16,fp8,0,0.023577600717544556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,12,128,1,fp8,fp8,0,0.023369599878787995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,1,128,1,float16,float16,0,0.02186879962682724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,1,128,1,float16,fp8,0,0.02282879948616028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,1,128,1,fp8,fp8,0,0.022702400386333466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,2,128,1,float16,float16,0,0.022078399360179902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,12,128,1,float16,fp8,0,0.02199520021677017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,2,128,1,float16,fp8,0,0.023163199424743652
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,2,128,1,fp8,fp8,0,0.022614400088787078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,4,128,1,float16,float16,0,0.022329600155353548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,4,128,1,float16,fp8,0,0.022815999388694764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,4,128,1,fp8,fp8,0,0.022720000147819518
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,12,128,1,float16,float16,0,0.0210207998752594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,12,128,1,fp8,fp8,0,0.021780799329280853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,1,128,1,float16,float16,0,0.02093919962644577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,1,128,1,float16,fp8,0,0.02178879976272583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,1,128,1,fp8,fp8,0,0.021499200165271758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,2,128,1,float16,float16,0,0.020931200683116914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,1,128,1,float16,float16,0,0.109825599193573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,2,128,1,float16,fp8,0,0.02166240066289902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,2,128,1,fp8,fp8,0,0.02158239930868149
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,4,128,1,float16,float16,0,0.020931200683116914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,2,128,1,float16,fp8,0,0.1331455945968628
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,4,128,1,float16,fp8,0,0.02173600047826767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,4,128,1,fp8,fp8,0,0.021886399388313292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,1,128,1,float16,fp8,0,0.1137503981590271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,1,128,1,fp8,fp8,0,0.11467679738998413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,2,128,1,float16,float16,0,0.1302175998687744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,2,128,1,fp8,fp8,0,0.13305120468139647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,4,128,1,float16,float16,0,0.18092479705810546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,4,128,1,float16,fp8,0,0.1869104027748108
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,4,128,1,fp8,fp8,0,0.18711520433425904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,12,128,1,float16,float16,0,0.19793280363082885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,12,128,1,float16,fp8,0,0.19581600427627563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,12,128,1,fp8,fp8,0,0.1968176007270813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,1,128,1,float16,float16,0,0.06819360256195069
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,1,128,1,float16,fp8,0,0.06723200082778931
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,1,128,1,fp8,fp8,0,0.06868159770965576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,2,128,1,float16,float16,0,0.07599040269851684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,2,128,1,float16,fp8,0,0.0767087996006012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,2,128,1,fp8,fp8,0,0.07710080146789551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,4,128,1,float16,float16,0,0.10339839458465576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,4,128,1,float16,fp8,0,0.10429919958114624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,4,128,1,fp8,fp8,0,0.10415840148925781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,12,128,1,float16,float16,0,0.10955040454864502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,12,128,1,float16,fp8,0,0.10497440099716186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,12,128,1,fp8,fp8,0,0.10509120225906372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,1,128,1,float16,float16,0,0.036657598614692685
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,1,128,1,float16,fp8,0,0.03695360124111176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,1,128,1,fp8,fp8,0,0.03711679875850678
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,2,128,1,float16,float16,0,0.0437279999256134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,2,128,1,float16,fp8,0,0.041863998770713805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,2,128,1,fp8,fp8,0,0.041715198755264284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,4,128,1,float16,float16,0,0.0605679988861084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,4,128,1,float16,fp8,0,0.05327039957046509
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,4,128,1,fp8,fp8,0,0.0541920006275177
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,12,128,1,float16,float16,0,0.06002399921417236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,12,128,1,float16,fp8,0,0.05392000079154968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,12,128,1,fp8,fp8,0,0.05460159778594971
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,1,128,1,float16,float16,0,0.028825598955154418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,1,128,1,float16,fp8,0,0.02977440059185028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,1,128,1,fp8,fp8,0,0.029945600032806396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,2,128,1,float16,float16,0,0.029497599601745604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,2,128,1,float16,fp8,0,0.030883198976516722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,2,128,1,fp8,fp8,0,0.03099200129508972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,4,128,1,float16,float16,0,0.03369120061397553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,4,128,1,float16,fp8,0,0.03485440015792847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,4,128,1,fp8,fp8,0,0.03471519947052002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,12,128,1,float16,float16,0,0.036051198840141296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,12,128,1,float16,fp8,0,0.037459200620651244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,12,128,1,fp8,fp8,0,0.037390398979187014
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,1,128,1,float16,float16,0,0.023548799753189086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,1,128,1,float16,fp8,0,0.024641600251197816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,1,128,1,fp8,fp8,0,0.02409279942512512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,2,128,1,float16,float16,0,0.02399040013551712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,2,128,1,float16,fp8,0,0.024857600033283234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,2,128,1,fp8,fp8,0,0.02499520033597946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,4,128,1,float16,float16,0,0.02462719976902008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,4,128,1,float16,fp8,0,0.025227200984954835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,4,128,1,fp8,fp8,0,0.025409600138664244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,12,128,1,float16,float16,0,0.025992000102996828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,12,128,1,float16,fp8,0,0.026657599210739135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,12,128,1,fp8,fp8,0,0.02697120010852814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,1,128,1,float16,float16,0,0.02338559925556183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,1,128,1,float16,fp8,0,0.024216000735759736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,1,128,1,fp8,fp8,0,0.024091200530529024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,2,128,1,float16,float16,0,0.023491199314594268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,2,128,1,float16,fp8,0,0.024087999761104584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,2,128,1,fp8,fp8,0,0.023955200612545014
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,4,128,1,float16,float16,0,0.02354719936847687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,4,128,1,float16,fp8,0,0.024167999625205994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,4,128,1,fp8,fp8,0,0.024396799504756927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,12,128,1,float16,float16,0,0.02226240038871765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,12,128,1,float16,fp8,0,0.023367999494075774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,12,128,1,fp8,fp8,0,0.023367999494075774
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,1,128,1,float16,float16,0,0.021980799734592438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,1,128,1,float16,fp8,0,0.022819200158119203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,1,128,1,fp8,fp8,0,0.022702400386333466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,2,128,1,float16,float16,0,0.02226399928331375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,2,128,1,float16,fp8,0,0.022771200537681578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,2,128,1,fp8,fp8,0,0.022729599475860597
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,4,128,1,float16,float16,0,0.021855999529361726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,4,128,1,float16,fp8,0,0.022620800137519836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,4,128,1,fp8,fp8,0,0.022668799757957457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,12,128,1,float16,float16,0,0.021321600675582884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,12,128,1,float16,fp8,0,0.021772800385951994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,12,128,1,fp8,fp8,0,0.022099199891090392
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,1,128,1,float16,float16,0,0.020905600488185884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,1,128,1,float16,fp8,0,0.021876800060272216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,1,128,1,fp8,fp8,0,0.021566399931907655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,2,128,1,float16,float16,0,0.020828799903392793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,2,128,1,float16,fp8,0,0.021780799329280853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,2,128,1,fp8,fp8,0,0.021614399552345277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,4,128,1,float16,float16,0,0.020934399962425233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,4,128,1,float16,fp8,0,0.021831999719142913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,4,128,1,fp8,fp8,0,0.02173919975757599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,12,128,1,float16,float16,0,0.02064319998025894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,12,128,1,float16,fp8,0,0.021400000154972076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,12,128,1,fp8,fp8,0,0.02136480063199997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,1,128,1,float16,float16,0,0.0205487996339798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,1,128,1,float16,fp8,0,0.02144320011138916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,1,128,1,fp8,fp8,0,0.021243199706077576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,2,128,1,float16,float16,0,0.02070239931344986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,2,128,1,float16,fp8,0,0.021433599293231964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,2,128,1,fp8,fp8,0,0.021476800739765167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,4,128,1,float16,float16,0,0.020603199303150178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,4,128,1,float16,fp8,0,0.021270400285720824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,4,128,1,fp8,fp8,0,0.021382400393486024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,1,128,1,float16,float16,0,0.06880159974098206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,1,128,1,float16,fp8,0,0.07002720236778259
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,1,128,1,fp8,fp8,0,0.0701744019985199
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,2,128,1,float16,float16,0,0.08434240221977234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,2,128,1,float16,fp8,0,0.08670880198478699
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,2,128,1,fp8,fp8,0,0.08662239909172058
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,4,128,1,float16,float16,0,0.1040992021560669
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,4,128,1,float16,fp8,0,0.1058575987815857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,4,128,1,fp8,fp8,0,0.10546560287475586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,12,128,1,float16,float16,0,0.13035039901733397
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,12,128,1,float16,fp8,0,0.13404799699783326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,12,128,1,fp8,fp8,0,0.1335968017578125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,1,128,1,float16,float16,0,0.036550399661064145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,1,128,1,float16,fp8,0,0.03772160112857818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,1,128,1,fp8,fp8,0,0.03812800049781799
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,2,128,1,float16,float16,0,0.047777599096298216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,2,128,1,float16,fp8,0,0.047203201055526736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,2,128,1,fp8,fp8,0,0.04738239943981171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,4,128,1,float16,float16,0,0.06112639904022217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,4,128,1,float16,fp8,0,0.055491197109222415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,4,128,1,fp8,fp8,0,0.055137598514556886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,12,128,1,float16,float16,0,0.07091040015220643
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,12,128,1,float16,fp8,0,0.06841599941253662
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,12,128,1,fp8,fp8,0,0.0682528018951416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,1,128,1,float16,float16,0,0.029236799478530882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,1,128,1,float16,fp8,0,0.030539199709892273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,1,128,1,fp8,fp8,0,0.03075999915599823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,2,128,1,float16,float16,0,0.029835200309753417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,2,128,1,float16,fp8,0,0.03160319924354553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,2,128,1,fp8,fp8,0,0.031446400284767154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,4,128,1,float16,float16,0,0.03366079926490784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,4,128,1,float16,fp8,0,0.03559840023517609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,4,128,1,fp8,fp8,0,0.035329601168632506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,12,128,1,float16,float16,0,0.040136000514030455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,12,128,1,float16,fp8,0,0.042444801330566405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,12,128,1,fp8,fp8,0,0.042552000284194945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,1,128,1,float16,float16,0,0.023999999463558196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,1,128,1,float16,fp8,0,0.024772800505161285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,1,128,1,fp8,fp8,0,0.02532159984111786
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,2,128,1,float16,float16,0,0.024299199879169463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,2,128,1,float16,fp8,0,0.0250575989484787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,2,128,1,fp8,fp8,0,0.024740800261497498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,4,128,1,float16,float16,0,0.02492479979991913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,4,128,1,float16,fp8,0,0.025433599948883057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,4,128,1,fp8,fp8,0,0.02563839852809906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,12,128,1,float16,float16,0,0.029817599058151244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,12,128,1,float16,fp8,0,0.031033599376678468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,12,128,1,fp8,fp8,0,0.03145279884338379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,1,128,1,float16,float16,0,0.023315200209617616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,1,128,1,float16,fp8,0,0.024401600658893585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,1,128,1,fp8,fp8,0,0.02396479994058609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,2,128,1,float16,float16,0,0.023127999901771546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,2,128,1,float16,fp8,0,0.024217599630355836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,12,128,1,float16,fp8,0,0.02330079972743988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,2,128,1,fp8,fp8,0,0.02396800071001053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,4,128,1,float16,float16,0,0.023604799807071686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,4,128,1,float16,fp8,0,0.02402079999446869
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,4,128,1,fp8,fp8,0,0.024167999625205994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,12,128,1,float16,float16,0,0.02239519953727722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,12,128,1,fp8,fp8,0,0.02354719936847687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,1,128,1,float16,float16,0,0.021887999773025513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,1,128,1,float16,fp8,0,0.02290560007095337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,1,128,1,fp8,fp8,0,0.02276639938354492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,2,128,1,float16,float16,0,0.021852800250053407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,2,128,1,float16,fp8,0,0.022932800650596618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,2,128,1,fp8,fp8,0,0.022873599827289582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,4,128,1,float16,float16,0,0.02176959961652756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,4,128,1,float16,fp8,0,0.02268480062484741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,4,128,1,fp8,fp8,0,0.02250239998102188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,12,128,1,float16,float16,0,0.021147200465202333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,12,128,1,float16,fp8,0,0.02204640060663223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,12,128,1,fp8,fp8,0,0.022111999988555908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,1,128,1,float16,float16,0,0.020956799387931824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,4,128,1,float16,fp8,0,0.022078399360179902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,1,128,1,float16,fp8,0,0.021422399580478667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,1,128,1,fp8,fp8,0,0.021875199675559998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,2,128,1,float16,float16,0,0.020985600352287293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,2,128,1,float16,fp8,0,0.021712000668048858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,2,128,1,fp8,fp8,0,0.021611200273036958
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,4,128,1,float16,float16,0,0.02096800059080124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,4,128,1,fp8,fp8,0,0.021609599888324737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,12,128,1,float16,float16,0,0.020828799903392793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,12,128,1,float16,fp8,0,0.021566399931907655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,12,128,1,fp8,fp8,0,0.021836799383163453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,1,128,1,float16,float16,0,0.020524799823760986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,1,128,1,float16,fp8,0,0.02134079933166504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,1,128,1,fp8,fp8,0,0.02158239930868149
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,2,128,1,float16,float16,0,0.020795199275016784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,2,128,1,float16,fp8,0,0.02134079933166504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,2,128,1,fp8,fp8,0,0.021382400393486024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,4,128,1,float16,float16,0,0.020310400426387785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,4,128,1,float16,fp8,0,0.021492800116539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,4,128,1,fp8,fp8,0,0.021147200465202333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,12,128,1,float16,float16,0,0.02052319943904877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,12,128,1,float16,fp8,0,0.021388800442218782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,2,128,1,fp8,fp8,0,0.021217599511146545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,12,128,1,fp8,fp8,0,0.02131199985742569
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,1,128,1,float16,float16,0,0.020483200252056123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,1,128,1,float16,fp8,0,0.02144320011138916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,1,128,1,fp8,fp8,0,0.021278400719165803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,2,128,1,float16,float16,0,0.020472000539302825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,2,128,1,float16,fp8,0,0.02115200012922287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,4,128,1,float16,float16,0,0.020343999564647674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,4,128,1,float16,fp8,0,0.021171200275421142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,4,128,1,fp8,fp8,0,0.021414400637149812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,1,128,1,float16,float16,0,0.04169279932975769
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,1,128,1,float16,fp8,0,0.04421919882297516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,4,128,1,float16,fp8,0,0.07499200105667114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,1,128,1,fp8,fp8,0,0.043996798992156985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,2,128,1,float16,float16,0,0.049958398938179015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,2,128,1,float16,fp8,0,0.049214398860931395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,12,128,1,fp8,fp8,0,0.0959280014038086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,2,128,1,fp8,fp8,0,0.048982399702072146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,4,128,1,float16,float16,0,0.07745440006256103
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,4,128,1,fp8,fp8,0,0.0745024025440216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,12,128,1,float16,float16,0,0.09571679830551147
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,12,128,1,float16,fp8,0,0.09625920057296752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,1,128,1,float16,float16,0,0.030008000135421754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,1,128,1,float16,fp8,0,0.03138880133628845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,1,128,1,fp8,fp8,0,0.03171359896659851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,2,128,1,float16,float16,0,0.030895999073982237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,2,128,1,float16,fp8,0,0.03226560056209564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,2,128,1,fp8,fp8,0,0.032201600074768064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,4,128,1,float16,float16,0,0.042403200268745424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,4,128,1,float16,fp8,0,0.045332801342010495
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,4,128,1,fp8,fp8,0,0.04550719857215881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,12,128,1,float16,float16,0,0.05188000202178955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,1,128,1,fp8,fp8,0,0.02529920041561127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,12,128,1,float16,fp8,0,0.056329602003097536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,12,128,1,fp8,fp8,0,0.05658239722251892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,1,128,1,float16,float16,0,0.024676799774169922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,1,128,1,float16,fp8,0,0.025155198574066163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,2,128,1,float16,float16,0,0.025180798768997193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,2,128,1,float16,fp8,0,0.025436800718307496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,2,128,1,fp8,fp8,0,0.02539680004119873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,4,128,1,float16,float16,0,0.029025599360466003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,4,128,1,float16,fp8,0,0.030793601274490358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,4,128,1,fp8,fp8,0,0.030788800120353697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,12,128,1,float16,float16,0,0.03402239978313446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,12,128,1,float16,fp8,0,0.03644160032272339
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,12,128,1,fp8,fp8,0,0.03592959940433502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,1,128,1,float16,float16,0,0.02401279956102371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,1,128,1,float16,fp8,0,0.024519999325275422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,1,128,1,fp8,fp8,0,0.024607999622821806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,2,128,1,float16,float16,0,0.0234607994556427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,2,128,1,float16,fp8,0,0.024395200610160827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,2,128,1,fp8,fp8,0,0.02460159957408905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,4,128,1,float16,float16,0,0.023712000250816344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,4,128,1,float16,fp8,0,0.024700799584388734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,4,128,1,fp8,fp8,0,0.024568000435829164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,12,128,1,float16,float16,0,0.02617279887199402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,12,128,1,float16,fp8,0,0.02741599977016449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,12,128,1,fp8,fp8,0,0.02754719853401184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,1,128,1,float16,float16,0,0.021799999475479125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,1,128,1,float16,fp8,0,0.022732800245285033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,1,128,1,fp8,fp8,0,0.022793599963188173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,2,128,1,float16,float16,0,0.022100800275802614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,2,128,1,float16,fp8,0,0.022886399924755097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,2,128,1,fp8,fp8,0,0.022524799406528472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,4,128,1,float16,float16,0,0.021876800060272216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,4,128,1,float16,fp8,0,0.022801600396633148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,4,128,1,fp8,fp8,0,0.022580799460411072
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,12,128,1,float16,float16,0,0.021414400637149812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,12,128,1,float16,fp8,0,0.022200000286102296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,12,128,1,fp8,fp8,0,0.022324800491333008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,1,128,1,float16,float16,0,0.021272000670433045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,1,128,1,float16,fp8,0,0.021827200055122377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,1,128,1,fp8,fp8,0,0.021887999773025513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,2,128,1,float16,float16,0,0.02114879935979843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,2,128,1,float16,fp8,0,0.021678400039672852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,12,128,1,fp8,fp8,0,0.021540799736976625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,2,128,1,fp8,fp8,0,0.0217071995139122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,4,128,1,float16,float16,0,0.02112479954957962
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,4,128,1,float16,fp8,0,0.02173600047826767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,4,128,1,fp8,fp8,0,0.022111999988555908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,12,128,1,float16,float16,0,0.02104319930076599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,12,128,1,float16,fp8,0,0.021723200380802155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,1,128,1,float16,float16,0,0.020684799551963805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,1,128,1,float16,fp8,0,0.021238400042057036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,1,128,1,fp8,fp8,0,0.02130720019340515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,2,128,1,float16,float16,0,0.02046239972114563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,2,128,1,float16,fp8,0,0.021030400693416596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,2,128,1,fp8,fp8,0,0.021382400393486024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,4,128,1,float16,float16,0,0.020580799877643587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,4,128,1,float16,fp8,0,0.021422399580478667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,4,128,1,fp8,fp8,0,0.021358400583267212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,12,128,1,float16,float16,0,0.02066880017518997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,12,128,1,float16,fp8,0,0.021352000534534454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,12,128,1,fp8,fp8,0,0.021675199270248413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,1,128,1,float16,float16,0,0.020420800149440765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,1,128,1,float16,fp8,0,0.021345600485801697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,1,128,1,fp8,fp8,0,0.021473599970340727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,2,128,1,float16,float16,0,0.020351999998092653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,2,128,1,float16,fp8,0,0.02112639993429184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,2,128,1,fp8,fp8,0,0.021222400665283202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,4,128,1,float16,float16,0,0.020267200469970704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,4,128,1,float16,fp8,0,0.021329599618911742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,4,128,1,fp8,fp8,0,0.02101600021123886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,12,128,1,float16,float16,0,0.02046400010585785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,12,128,1,float16,fp8,0,0.021201600134372712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,12,128,1,fp8,fp8,0,0.02134400010108948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,1,128,1,float16,float16,0,0.020259200036525725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,1,128,1,float16,fp8,0,0.021164800226688384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,1,128,1,fp8,fp8,0,0.021089600026607515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,2,128,1,float16,float16,0,0.020095999538898467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,2,128,1,float16,fp8,0,0.021140800416469575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,2,128,1,fp8,fp8,0,0.021078400313854218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,4,128,1,float16,float16,0,0.020436799526214598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,4,128,1,float16,fp8,0,0.02141920030117035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,4,128,1,fp8,fp8,0,0.021011200547218323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,1,128,1,float16,fp8,0,6.340742492675782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,1,128,1,fp8,fp8,0,6.39826545715332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,2,128,1,float16,fp8,0,6.472700500488282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,2,128,1,fp8,fp8,0,6.474900817871093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,1,128,1,float16,float16,0,7.724267578125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,2,128,1,float16,float16,0,7.5863807678222654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,4,128,1,float16,fp8,0,6.7938079833984375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,4,128,1,float16,float16,0,7.681918334960938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,8,128,1,float16,fp8,0,3.676927947998047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,8,128,1,float16,float16,0,4.258561706542968
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,8,128,1,fp8,fp8,0,3.69775505065918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,1,128,1,float16,fp8,0,3.1998367309570312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,1,128,1,fp8,fp8,0,3.16232967376709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,1,128,1,float16,float16,0,3.770131301879883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,2,128,1,float16,float16,0,3.7317249298095705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,4,128,1,fp8,fp8,0,6.782788848876953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,2,128,1,float16,fp8,0,3.24743537902832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,2,128,1,fp8,fp8,0,3.2700336456298826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,8,128,1,float16,float16,0,2.081060791015625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,4,128,1,float16,fp8,0,3.388671875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,4,128,1,fp8,fp8,0,3.40423698425293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,4,128,1,float16,float16,0,3.8281982421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,8,128,1,float16,fp8,0,1.8537263870239258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,8,128,1,fp8,fp8,0,1.8619712829589843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,1,128,1,float16,float16,0,1.756537628173828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,1,128,1,float16,fp8,0,1.6342256546020508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,1,128,1,fp8,fp8,0,1.5848624229431152
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,2,128,1,float16,float16,0,1.8604911804199218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,2,128,1,float16,fp8,0,1.771059226989746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,2,128,1,fp8,fp8,0,1.624760055541992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,8,128,1,float16,float16,0,1.0090271949768066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,8,128,1,float16,fp8,0,1.0957856178283691
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,8,128,1,fp8,fp8,0,0.9792544364929199
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,4,128,1,fp8,fp8,0,1.7223264694213867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,1,128,1,float16,float16,0,0.8722640037536621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,4,128,1,float16,float16,0,1.8922895431518554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,4,128,1,float16,fp8,0,1.8315391540527344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,1,128,1,float16,fp8,0,0.8597264289855957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,1,128,1,fp8,fp8,0,0.8784848213195801
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,2,128,1,float16,fp8,0,0.8563695907592773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,2,128,1,float16,float16,0,0.9065728187561035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,2,128,1,fp8,fp8,0,0.8454655647277832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,4,128,1,float16,float16,0,0.9420096397399902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,4,128,1,float16,fp8,0,0.9801504135131835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,4,128,1,fp8,fp8,0,0.8998959541320801
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,1,128,1,float16,fp8,0,3.638409423828125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,1,128,1,fp8,fp8,0,3.632215881347656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,1,128,1,float16,float16,0,4.31830062866211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,2,128,1,float16,fp8,0,3.780628967285156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,2,128,1,fp8,fp8,0,3.926599884033203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,2,128,1,float16,float16,0,4.484172821044922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,4,128,1,float16,fp8,0,4.029385757446289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,4,128,1,float16,float16,0,4.699959945678711
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,8,128,1,float16,float16,0,2.527934455871582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,8,128,1,float16,fp8,0,2.2961151123046877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,8,128,1,fp8,fp8,0,2.228993606567383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,1,128,1,float16,float16,0,1.981007957458496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,1,128,1,float16,fp8,0,1.9760656356811523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,1,128,1,fp8,fp8,0,1.8631856918334961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,2,128,1,float16,float16,0,2.1402639389038085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,4,128,1,fp8,fp8,0,4.002897644042969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,2,128,1,fp8,fp8,0,1.8857263565063476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,2,128,1,float16,fp8,0,2.009464073181152
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,8,128,1,float16,float16,0,1.246720027923584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,8,128,1,fp8,fp8,0,1.1323967933654786
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,4,128,1,float16,float16,0,2.248219108581543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,8,128,1,float16,fp8,0,1.2849360466003419
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,4,128,1,fp8,fp8,0,2.0139631271362304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,4,128,1,float16,fp8,0,2.052004814147949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,1,128,1,float16,float16,0,0.9918736457824707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,1,128,1,float16,fp8,0,0.9625887870788574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,1,128,1,fp8,fp8,0,0.9402064323425293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,2,128,1,float16,float16,0,1.0158687591552735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,2,128,1,float16,fp8,0,1.022655963897705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,2,128,1,fp8,fp8,0,1.016380786895752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,4,128,1,float16,float16,0,1.1252592086791993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,4,128,1,float16,fp8,0,1.0392304420471192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,8,128,1,float16,float16,0,0.6619679927825928
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,8,128,1,float16,fp8,0,0.595300817489624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,8,128,1,fp8,fp8,0,0.5985007762908936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,4,128,1,fp8,fp8,0,1.0869312286376953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,1,128,1,float16,float16,0,0.5114816188812256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,1,128,1,float16,fp8,0,0.5110864162445068
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,1,128,1,fp8,fp8,0,0.5356480121612549
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,2,128,1,float16,float16,0,0.5229375839233399
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,4,128,1,float16,float16,0,0.5650864124298096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,2,128,1,float16,fp8,0,0.4969776153564453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,4,128,1,fp8,fp8,0,0.5319136142730713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,2,128,1,fp8,fp8,0,0.5067247867584228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,4,128,1,float16,fp8,0,0.5453648090362548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,1,128,1,float16,fp8,0,2.659390449523926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,1,128,1,float16,float16,0,2.9334272384643554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,1,128,1,fp8,fp8,0,2.615732765197754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,2,128,1,float16,fp8,0,2.712049674987793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,2,128,1,float16,float16,0,3.088484764099121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,2,128,1,fp8,fp8,0,2.694699287414551
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,4,128,1,float16,fp8,0,2.9193088531494142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,4,128,1,float16,float16,0,3.2211582183837892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,8,128,1,float16,fp8,0,1.6382272720336915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,1,128,1,float16,fp8,0,1.3207823753356933
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,8,128,1,float16,float16,0,1.8526031494140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,1,128,1,float16,float16,0,1.5382304191589355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,8,128,1,fp8,fp8,0,1.6548175811767578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,1,128,1,fp8,fp8,0,1.3031647682189942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,4,128,1,fp8,fp8,0,2.864001655578613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,2,128,1,float16,float16,0,1.543609619140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,2,128,1,float16,fp8,0,1.3492783546447753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,2,128,1,fp8,fp8,0,1.353001594543457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,8,128,1,float16,float16,0,0.9191472053527832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,8,128,1,float16,fp8,0,0.8439760208129883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,4,128,1,float16,float16,0,1.5926128387451173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,4,128,1,float16,fp8,0,1.490777587890625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,8,128,1,fp8,fp8,0,0.8961312294006347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,4,128,1,fp8,fp8,0,1.4476592063903808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,1,128,1,float16,float16,0,0.7086751937866211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,1,128,1,float16,fp8,0,0.6699632167816162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,1,128,1,fp8,fp8,0,0.669385576248169
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,2,128,1,float16,float16,0,0.7473872184753418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,2,128,1,float16,fp8,0,0.726859188079834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,2,128,1,fp8,fp8,0,0.6943056106567382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,4,128,1,float16,float16,0,0.8050463676452637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,4,128,1,float16,fp8,0,0.748528003692627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,8,128,1,float16,float16,0,0.4999887943267822
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,8,128,1,float16,fp8,0,0.45427842140197755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,4,128,1,fp8,fp8,0,0.7407936096191406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,8,128,1,fp8,fp8,0,0.45893278121948244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,1,128,1,float16,float16,0,0.3693311929702759
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,1,128,1,float16,fp8,0,0.35319840908050537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,1,128,1,fp8,fp8,0,0.35312480926513673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,2,128,1,float16,float16,0,0.38719840049743653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,2,128,1,float16,fp8,0,0.3711999893188477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,2,128,1,fp8,fp8,0,0.36550400257110593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,4,128,1,float16,float16,0,0.4229248046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,4,128,1,float16,fp8,0,0.39452319145202636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,4,128,1,fp8,fp8,0,0.39485599994659426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,1,128,1,float16,fp8,0,3.394051361083984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,1,128,1,fp8,fp8,0,3.4309326171875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,1,128,1,float16,float16,0,3.9085247039794924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,2,128,1,float16,fp8,0,3.57193603515625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,2,128,1,fp8,fp8,0,3.5382495880126954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,2,128,1,float16,float16,0,4.057585525512695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,4,128,1,float16,float16,0,4.311057662963867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,8,128,1,float16,float16,0,2.48624324798584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,8,128,1,float16,fp8,0,2.340577507019043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,1,128,1,float16,float16,0,1.8281200408935547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,4,128,1,float16,fp8,0,3.823731231689453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,1,128,1,float16,fp8,0,1.769620704650879
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,8,128,1,fp8,fp8,0,2.210531234741211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,1,128,1,fp8,fp8,0,1.848428726196289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,4,128,1,fp8,fp8,0,3.798263931274414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,2,128,1,float16,fp8,0,1.775107192993164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,2,128,1,float16,float16,0,2.0284671783447266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,2,128,1,fp8,fp8,0,1.8889568328857422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,8,128,1,float16,fp8,0,1.1106271743774414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,8,128,1,float16,float16,0,1.2049712181091308
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,4,128,1,float16,fp8,0,1.924323272705078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,4,128,1,float16,float16,0,2.132975959777832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,4,128,1,fp8,fp8,0,1.9241840362548828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,1,128,1,float16,float16,0,0.9780336380004883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,8,128,1,fp8,fp8,0,1.1224608421325684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,1,128,1,float16,fp8,0,0.8936592102050781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,1,128,1,fp8,fp8,0,0.8545696258544921
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,2,128,1,float16,float16,0,0.9623264312744141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,2,128,1,float16,fp8,0,0.9116335868835449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,2,128,1,fp8,fp8,0,0.974953556060791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,4,128,1,float16,float16,0,1.047092819213867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,8,128,1,float16,float16,0,0.6054992198944091
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,8,128,1,float16,fp8,0,0.5678575992584228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,4,128,1,float16,fp8,0,1.0602319717407227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,4,128,1,fp8,fp8,0,0.9944463729858398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,8,128,1,fp8,fp8,0,0.5865231990814209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,1,128,1,float16,float16,0,0.47239198684692385
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,1,128,1,float16,fp8,0,0.47374401092529295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,1,128,1,fp8,fp8,0,0.4735407829284668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,2,128,1,float16,float16,0,0.48781437873840333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,2,128,1,float16,fp8,0,0.4651440143585205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,2,128,1,fp8,fp8,0,0.48605761528015134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,4,128,1,float16,float16,0,0.534603214263916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,4,128,1,float16,fp8,0,0.5090271949768066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,8,128,1,float16,float16,0,0.32321920394897463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,4,128,1,fp8,fp8,0,0.5084943771362305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,8,128,1,float16,fp8,0,0.3124799966812134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,8,128,1,fp8,fp8,0,0.2988703966140747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,1,128,1,float16,float16,0,0.24123680591583252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,1,128,1,float16,fp8,0,0.23274080753326415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,1,128,1,fp8,fp8,0,0.23384320735931396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,4,128,1,float16,fp8,0,0.2645951986312866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,2,128,1,float16,float16,0,0.2564079999923706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,2,128,1,float16,fp8,0,0.24242238998413085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,2,128,1,fp8,fp8,0,0.2455904006958008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,4,128,1,float16,float16,0,0.28060159683227537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,4,128,1,fp8,fp8,0,0.25951519012451174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,1,128,1,float16,float16,0,2.233700752258301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,1,128,1,float16,fp8,0,2.0125423431396485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,1,128,1,fp8,fp8,0,2.0140079498291015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,2,128,1,float16,fp8,0,2.1283391952514648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,2,128,1,float16,float16,0,2.324177551269531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,2,128,1,fp8,fp8,0,2.1266159057617187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,4,128,1,float16,fp8,0,2.379884719848633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,4,128,1,float16,float16,0,2.5768304824829102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,8,128,1,float16,float16,0,1.5181743621826171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,1,128,1,float16,fp8,0,1.0064592361450195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,8,128,1,float16,fp8,0,1.399841594696045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,1,128,1,float16,float16,0,1.1085647583007812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,8,128,1,fp8,fp8,0,1.420201587677002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,1,128,1,fp8,fp8,0,1.0330495834350586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,4,128,1,fp8,fp8,0,2.4812463760375976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,2,128,1,float16,float16,0,1.2339200019836425
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,2,128,1,float16,fp8,0,1.089409637451172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,2,128,1,fp8,fp8,0,1.057532787322998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,4,128,1,float16,float16,0,1.2496560096740723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,8,128,1,float16,float16,0,0.7640063762664795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,8,128,1,float16,fp8,0,0.7300672054290771
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,4,128,1,float16,fp8,0,1.2301376342773438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,4,128,1,fp8,fp8,0,1.188262367248535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,1,128,1,float16,float16,0,0.5483983993530274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,8,128,1,fp8,fp8,0,0.751913595199585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,1,128,1,float16,fp8,0,0.5180880069732666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,1,128,1,fp8,fp8,0,0.517848014831543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,2,128,1,float16,float16,0,0.5672048091888428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,2,128,1,float16,fp8,0,0.553327989578247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,2,128,1,fp8,fp8,0,0.5448448181152343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,4,128,1,float16,float16,0,0.6427103996276855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,4,128,1,float16,fp8,0,0.6225152015686035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,4,128,1,fp8,fp8,0,0.6099408149719239
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,8,128,1,float16,float16,0,0.412886381149292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,8,128,1,float16,fp8,0,0.3711967945098877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,8,128,1,fp8,fp8,0,0.3795919895172119
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,1,128,1,float16,float16,0,0.2811471939086914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,1,128,1,float16,fp8,0,0.27734079360961916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,1,128,1,fp8,fp8,0,0.27647199630737307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,2,128,1,float16,float16,0,0.3010031938552856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,2,128,1,float16,fp8,0,0.29082560539245605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,2,128,1,fp8,fp8,0,0.29135680198669434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,4,128,1,float16,float16,0,0.34010560512542726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,4,128,1,float16,fp8,0,0.31884160041809084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,8,128,1,fp8,fp8,0,0.20446879863739015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,4,128,1,fp8,fp8,0,0.31832640171051024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,8,128,1,float16,float16,0,0.22768158912658693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,8,128,1,float16,fp8,0,0.20426878929138184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,1,128,1,float16,float16,0,0.15971360206604004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,1,128,1,float16,fp8,0,0.14881279468536376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,1,128,1,fp8,fp8,0,0.1492975950241089
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,2,128,1,float16,float16,0,0.17018239498138427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,2,128,1,float16,fp8,0,0.15596959590911866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,2,128,1,fp8,fp8,0,0.15632799863815308
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,4,128,1,float16,float16,0,0.18583840131759644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,4,128,1,float16,fp8,0,0.17106239795684813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,4,128,1,fp8,fp8,0,0.17043839693069457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,1,128,1,float16,float16,0,2.0249568939208986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,1,128,1,float16,fp8,0,1.933451271057129
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,1,128,1,fp8,fp8,0,1.9296079635620118
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,2,128,1,float16,float16,0,2.245280075073242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,2,128,1,float16,fp8,0,2.0641056060791017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,2,128,1,fp8,fp8,0,2.079947280883789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,4,128,1,float16,float16,0,2.5577808380126954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,8,128,1,float16,float16,0,1.5837344169616698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,1,128,1,float16,float16,0,1.028115177154541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,1,128,1,float16,fp8,0,0.9814847946166992
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,8,128,1,float16,fp8,0,1.513366413116455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,8,128,1,fp8,fp8,0,1.6158943176269531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,4,128,1,float16,fp8,0,2.437188720703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,4,128,1,fp8,fp8,0,2.391971206665039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,1,128,1,fp8,fp8,0,1.040668773651123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,2,128,1,float16,float16,0,1.1085599899291991
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,2,128,1,float16,fp8,0,1.0470416069030761
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,2,128,1,fp8,fp8,0,1.0462032318115235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,8,128,1,float16,float16,0,0.8002943992614746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,8,128,1,float16,fp8,0,0.7798463821411132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,4,128,1,float16,float16,0,1.2658191680908204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,4,128,1,float16,fp8,0,1.2358336448669434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,4,128,1,fp8,fp8,0,1.188974380493164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,1,128,1,float16,float16,0,0.5251696109771729
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,1,128,1,float16,fp8,0,0.4950128078460693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,8,128,1,fp8,fp8,0,0.7726255893707276
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,1,128,1,fp8,fp8,0,0.5047840118408203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,2,128,1,float16,float16,0,0.569547176361084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,2,128,1,float16,fp8,0,0.5431295871734619
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,2,128,1,fp8,fp8,0,0.5421232223510742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,4,128,1,float16,float16,0,0.6398560047149658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,4,128,1,float16,fp8,0,0.6167920112609864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,4,128,1,fp8,fp8,0,0.6051040172576905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,8,128,1,float16,float16,0,0.4145520210266113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,8,128,1,float16,fp8,0,0.3957279920578003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,8,128,1,fp8,fp8,0,0.39659359455108645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,1,128,1,float16,float16,0,0.27411839962005613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,1,128,1,float16,fp8,0,0.26785759925842284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,1,128,1,fp8,fp8,0,0.2639519929885864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,2,128,1,float16,float16,0,0.2949743986129761
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,2,128,1,float16,fp8,0,0.2870847940444946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,2,128,1,fp8,fp8,0,0.285425591468811
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,4,128,1,float16,float16,0,0.33355839252471925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,4,128,1,float16,fp8,0,0.3220992088317871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,4,128,1,fp8,fp8,0,0.32159359455108644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,8,128,1,float16,float16,0,0.22118079662322998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,2,128,1,float16,float16,0,0.16030240058898926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,8,128,1,float16,fp8,0,0.206823992729187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,8,128,1,fp8,fp8,0,0.20974559783935548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,1,128,1,float16,float16,0,0.15015360116958618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,1,128,1,float16,fp8,0,0.137881600856781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,1,128,1,fp8,fp8,0,0.13662879467010497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,2,128,1,float16,fp8,0,0.15188319683074952
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,2,128,1,fp8,fp8,0,0.15168479681015015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,4,128,1,float16,float16,0,0.18093600273132324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,4,128,1,float16,fp8,0,0.17187039852142333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,4,128,1,fp8,fp8,0,0.17242879867553712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,8,128,1,float16,float16,0,0.11856000423431397
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,8,128,1,float16,fp8,0,0.10903040170669556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,8,128,1,fp8,fp8,0,0.10750880241394042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,2,128,1,fp8,fp8,0,0.08038079738616943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,1,128,1,float16,float16,0,0.07862079739570618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,1,128,1,float16,fp8,0,0.07656000256538391
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,1,128,1,fp8,fp8,0,0.07599520087242126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,2,128,1,float16,float16,0,0.08419839739799499
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,2,128,1,float16,fp8,0,0.08010560274124146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,4,128,1,float16,float16,0,0.09463199973106384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,4,128,1,float16,fp8,0,0.08858399987220764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,4,128,1,fp8,fp8,0,0.08881120085716247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,1,128,1,float16,float16,0,1.2133328437805175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,1,128,1,float16,fp8,0,1.176252841949463
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,1,128,1,fp8,fp8,0,1.1785375595092773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,2,128,1,float16,float16,0,1.3372896194458008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,2,128,1,float16,fp8,0,1.2801936149597168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,2,128,1,fp8,fp8,0,1.2725152015686034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,4,128,1,float16,float16,0,1.5617664337158204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,8,128,1,float16,float16,0,1.0527471542358398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,4,128,1,float16,fp8,0,1.5142671585083007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,1,128,1,float16,float16,0,0.6147679805755615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,4,128,1,fp8,fp8,0,1.522590446472168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,8,128,1,float16,fp8,0,1.0374176025390625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,8,128,1,fp8,fp8,0,0.9917823791503906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,1,128,1,float16,fp8,0,0.6288544178009033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,1,128,1,fp8,fp8,0,0.6020336151123047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,2,128,1,float16,float16,0,0.676862382888794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,2,128,1,float16,fp8,0,0.6493968009948731
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,2,128,1,fp8,fp8,0,0.6463823795318604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,4,128,1,float16,float16,0,0.8038623809814454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,4,128,1,float16,fp8,0,0.7693183898925782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,8,128,1,float16,float16,0,0.5404736042022705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,4,128,1,fp8,fp8,0,0.7870992183685303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,8,128,1,float16,fp8,0,0.5116447925567627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,1,128,1,float16,float16,0,0.3196063995361328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,8,128,1,fp8,fp8,0,0.5088560104370117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,1,128,1,float16,fp8,0,0.31602399349212645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,1,128,1,fp8,fp8,0,0.3159008026123047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,2,128,1,float16,float16,0,0.35503039360046384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,2,128,1,float16,fp8,0,0.34930078983306884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,2,128,1,fp8,fp8,0,0.34352960586547854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,4,128,1,float16,float16,0,0.4083439826965332
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,4,128,1,float16,fp8,0,0.3976495981216431
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,1,128,1,float16,fp8,0,0.17091039419174195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,4,128,1,fp8,fp8,0,0.4002943992614746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,8,128,1,float16,fp8,0,0.2671119928359985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,2,128,1,float16,float16,0,0.18857920169830322
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,8,128,1,float16,float16,0,0.28487839698791506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,8,128,1,fp8,fp8,0,0.26675679683685305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,1,128,1,float16,float16,0,0.17467520236968995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,1,128,1,fp8,fp8,0,0.16802239418029785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,2,128,1,float16,fp8,0,0.18508000373840333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,2,128,1,fp8,fp8,0,0.18568799495697022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,4,128,1,float16,float16,0,0.2193824052810669
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,4,128,1,float16,fp8,0,0.21129119396209717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,4,128,1,fp8,fp8,0,0.2108720064163208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,8,128,1,float16,float16,0,0.15767840147018433
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,8,128,1,float16,fp8,0,0.14783840179443358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,8,128,1,fp8,fp8,0,0.14629119634628296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,1,128,1,float16,float16,0,0.09588959813117981
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,1,128,1,float16,fp8,0,0.09196640253067016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,1,128,1,fp8,fp8,0,0.09142559766769409
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,2,128,1,float16,float16,0,0.10589599609375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,2,128,1,float16,fp8,0,0.09749760031700135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,2,128,1,fp8,fp8,0,0.09768800139427185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,4,128,1,float16,float16,0,0.12247200012207031
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,4,128,1,float16,fp8,0,0.1115407943725586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,4,128,1,fp8,fp8,0,0.11331039667129517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,8,128,1,float16,float16,0,0.08878560066223144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,8,128,1,float16,fp8,0,0.08117120265960694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,8,128,1,fp8,fp8,0,0.08032799959182739
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,1,128,1,float16,float16,0,0.06069120168685913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,1,128,1,float16,fp8,0,0.06064320206642151
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,1,128,1,fp8,fp8,0,0.060811197757720946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,2,128,1,float16,float16,0,0.06336960196495056
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,2,128,1,float16,fp8,0,0.06369760036468505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,2,128,1,fp8,fp8,0,0.06353440284729003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,4,128,1,float16,float16,0,0.0698527991771698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,4,128,1,float16,fp8,0,0.0692192018032074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,4,128,1,fp8,fp8,0,0.06918879747390747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,1,128,1,float16,float16,0,1.2237744331359863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,1,128,1,float16,fp8,0,1.1838288307189941
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,1,128,1,fp8,fp8,0,1.177582359313965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,2,128,1,float16,float16,0,1.367353630065918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,2,128,1,float16,fp8,0,1.337939167022705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,2,128,1,fp8,fp8,0,1.3142592430114746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,4,128,1,float16,float16,0,1.685433578491211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,8,128,1,float16,float16,0,1.1613103866577148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,4,128,1,float16,fp8,0,1.635291290283203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,8,128,1,float16,fp8,0,1.1209983825683594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,1,128,1,float16,float16,0,0.6236192226409912
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,1,128,1,float16,fp8,0,0.5989391803741455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,4,128,1,fp8,fp8,0,1.7143680572509765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,8,128,1,fp8,fp8,0,1.129862403869629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,1,128,1,fp8,fp8,0,0.6086448192596435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,2,128,1,float16,float16,0,0.6922624111175537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,2,128,1,float16,fp8,0,0.67467360496521
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,2,128,1,fp8,fp8,0,0.6689407825469971
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,4,128,1,float16,float16,0,0.8596032142639161
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,4,128,1,float16,fp8,0,0.8237536430358887
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,8,128,1,float16,float16,0,0.5990384101867676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,1,128,1,float16,float16,0,0.3206048011779785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,4,128,1,fp8,fp8,0,0.8408927917480469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,8,128,1,float16,fp8,0,0.5812784194946289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,1,128,1,float16,fp8,0,0.3128175973892212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,8,128,1,fp8,fp8,0,0.5721888065338134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,1,128,1,fp8,fp8,0,0.3168447971343994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,4,128,1,float16,float16,0,0.44583520889282224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,2,128,1,float16,float16,0,0.3610896110534668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,2,128,1,float16,fp8,0,0.3516783952713013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,2,128,1,fp8,fp8,0,0.3468832015991211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,8,128,1,float16,fp8,0,0.30225439071655275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,4,128,1,float16,fp8,0,0.42972640991210936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,4,128,1,fp8,fp8,0,0.422811222076416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,8,128,1,float16,float16,0,0.31388800144195556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,8,128,1,fp8,fp8,0,0.3012655973434448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,1,128,1,float16,float16,0,0.1747167944908142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,1,128,1,float16,fp8,0,0.16946719884872435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,1,128,1,fp8,fp8,0,0.17252639532089234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,2,128,1,float16,float16,0,0.19614880084991454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,2,128,1,float16,fp8,0,0.19030239582061767
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,2,128,1,fp8,fp8,0,0.19020639657974242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,4,128,1,float16,float16,0,0.23601279258728028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,4,128,1,float16,fp8,0,0.22637441158294677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,4,128,1,fp8,fp8,0,0.2234368085861206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,8,128,1,float16,float16,0,0.1693552017211914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,8,128,1,float16,fp8,0,0.16336640119552612
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,2,128,1,float16,float16,0,0.10615999698638916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,8,128,1,fp8,fp8,0,0.16180479526519775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,1,128,1,float16,float16,0,0.09869120121002198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,1,128,1,float16,fp8,0,0.09288319945335388
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,1,128,1,fp8,fp8,0,0.09301599860191345
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,2,128,1,float16,fp8,0,0.10235680341720581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,2,128,1,fp8,fp8,0,0.10236320495605469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,4,128,1,float16,float16,0,0.1258687973022461
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,4,128,1,float16,fp8,0,0.12337280511856079
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,4,128,1,fp8,fp8,0,0.12331680059432984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,8,128,1,float16,float16,0,0.09379680156707763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,8,128,1,float16,fp8,0,0.08737760186195373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,8,128,1,fp8,fp8,0,0.08550400137901307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,1,128,1,float16,float16,0,0.05395519733428955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,1,128,1,float16,fp8,0,0.05216320157051087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,1,128,1,fp8,fp8,0,0.05330079793930054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,2,128,1,float16,float16,0,0.058526402711868285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,2,128,1,float16,fp8,0,0.057873600721359254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,2,128,1,fp8,fp8,0,0.058380800485610965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,4,128,1,float16,float16,0,0.06902719736099243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,4,128,1,float16,fp8,0,0.06579359769821166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,4,128,1,fp8,fp8,0,0.06353600025177002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,8,128,1,float16,float16,0,0.06047840118408203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,8,128,1,float16,fp8,0,0.05921279788017273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,8,128,1,fp8,fp8,0,0.05938400030136108
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,1,128,1,float16,float16,0,0.047654399275779726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,1,128,1,float16,fp8,0,0.047491198778152464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,1,128,1,fp8,fp8,0,0.047865599393844604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,2,128,1,float16,float16,0,0.04804799854755402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,2,128,1,float16,fp8,0,0.047732800245285034
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,2,128,1,fp8,fp8,0,0.047777599096298216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,4,128,1,float16,float16,0,0.05255200266838074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,4,128,1,float16,fp8,0,0.05216159820556641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,4,128,1,fp8,fp8,0,0.053548800945281985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,1,128,1,float16,float16,0,0.8540495872497559
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,1,128,1,float16,fp8,0,0.8246912002563477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,1,128,1,fp8,fp8,0,0.8249327659606933
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,2,128,1,float16,float16,0,0.9633184432983398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,2,128,1,fp8,fp8,0,0.9414815902709961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,2,128,1,float16,fp8,0,0.9381423950195312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,4,128,1,float16,float16,0,1.1994688034057617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,8,128,1,float16,float16,0,0.8437328338623047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,4,128,1,float16,fp8,0,1.172873592376709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,8,128,1,float16,fp8,0,0.8199007987976075
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,4,128,1,fp8,fp8,0,1.161734390258789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,1,128,1,float16,float16,0,0.4401711940765381
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,8,128,1,fp8,fp8,0,0.8246463775634766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,1,128,1,float16,fp8,0,0.4303296089172363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,1,128,1,fp8,fp8,0,0.42973761558532714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,2,128,1,float16,float16,0,0.4920720100402832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,2,128,1,float16,fp8,0,0.4836991786956787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,2,128,1,fp8,fp8,0,0.4792736053466797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,4,128,1,float16,float16,0,0.6120736122131347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,4,128,1,float16,fp8,0,0.5935696125030517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,8,128,1,float16,float16,0,0.44255361557006834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,4,128,1,fp8,fp8,0,0.6019216060638428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,8,128,1,fp8,fp8,0,0.422819185256958
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,8,128,1,float16,fp8,0,0.42760319709777833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,1,128,1,float16,float16,0,0.23162400722503662
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,1,128,1,float16,fp8,0,0.22818400859832763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,1,128,1,fp8,fp8,0,0.22772960662841796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,2,128,1,float16,float16,0,0.25823678970336916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,2,128,1,float16,fp8,0,0.25373919010162355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,2,128,1,fp8,fp8,0,0.25796639919281006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,4,128,1,float16,float16,0,0.3189807891845703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,4,128,1,float16,fp8,0,0.31478559970855713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,4,128,1,fp8,fp8,0,0.31144959926605226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,8,128,1,float16,float16,0,0.2357759952545166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,8,128,1,float16,fp8,0,0.22704639434814453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,8,128,1,fp8,fp8,0,0.22329440116882324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,1,128,1,float16,float16,0,0.1279744029045105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,1,128,1,float16,fp8,0,0.12929600477218628
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,1,128,1,fp8,fp8,0,0.12974239587783815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,2,128,1,float16,float16,0,0.1446463942527771
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,4,128,1,fp8,fp8,0,0.1694543957710266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,2,128,1,float16,fp8,0,0.14131040573120118
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,2,128,1,fp8,fp8,0,0.1430848002433777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,4,128,1,float16,float16,0,0.17539520263671876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,4,128,1,float16,fp8,0,0.16925599575042724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,8,128,1,float16,float16,0,0.1300480008125305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,8,128,1,float16,fp8,0,0.12594560384750367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,8,128,1,fp8,fp8,0,0.1265104055404663
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,1,128,1,float16,float16,0,0.07177600264549255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,1,128,1,float16,fp8,0,0.06991999745368957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,4,128,1,float16,float16,0,0.09896799921989441
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,1,128,1,fp8,fp8,0,0.07074880003929138
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,2,128,1,float16,float16,0,0.08249120116233825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,2,128,1,float16,fp8,0,0.07609919905662536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,2,128,1,fp8,fp8,0,0.08087999820709228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,4,128,1,float16,fp8,0,0.09337440133094788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,4,128,1,fp8,fp8,0,0.09282240271568298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,8,128,1,float16,float16,0,0.07351040244102477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,8,128,1,float16,fp8,0,0.06571199893951415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,8,128,1,fp8,fp8,0,0.0659600019454956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,1,128,1,float16,float16,0,0.04485119879245758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,1,128,1,float16,fp8,0,0.04654400050640106
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,1,128,1,fp8,fp8,0,0.0457152009010315
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,2,128,1,float16,float16,0,0.04771040081977844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,2,128,1,float16,fp8,0,0.048497599363327024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,2,128,1,fp8,fp8,0,0.04811840057373047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,4,128,1,float16,float16,0,0.054833602905273435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,4,128,1,float16,fp8,0,0.054225599765777587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,4,128,1,fp8,fp8,0,0.05522239804267883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,8,128,1,float16,float16,0,0.06164000034332275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,8,128,1,float16,fp8,0,0.061508798599243165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,8,128,1,fp8,fp8,0,0.0594976007938385
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,1,128,1,float16,float16,0,0.04458079934120178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,1,128,1,float16,fp8,0,0.0455487996339798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,1,128,1,fp8,fp8,0,0.045075199007987975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,2,128,1,float16,float16,0,0.04829440116882324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,2,128,1,float16,fp8,0,0.04960319995880127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,2,128,1,fp8,fp8,0,0.04897280037403107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,4,128,1,float16,float16,0,0.05183839797973633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,4,128,1,float16,fp8,0,0.05311999917030334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,4,128,1,fp8,fp8,0,0.05350559949874878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,1,128,1,float16,float16,0,0.9235695838928223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,1,128,1,float16,fp8,0,0.9090352058410645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,1,128,1,fp8,fp8,0,0.9024831771850585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,2,128,1,float16,float16,0,1.0717071533203124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,2,128,1,float16,fp8,0,1.0568960189819336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,2,128,1,fp8,fp8,0,1.0497695922851562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,4,128,1,float16,float16,0,1.3929648399353027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,8,128,1,float16,float16,0,1.022049617767334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,4,128,1,float16,fp8,0,1.3752559661865233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,1,128,1,float16,float16,0,0.47129597663879397
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,4,128,1,fp8,fp8,0,1.3603887557983398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,8,128,1,float16,fp8,0,0.995361614227295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,8,128,1,fp8,fp8,0,1.000921630859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,1,128,1,float16,fp8,0,0.47053117752075196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,1,128,1,fp8,fp8,0,0.4679887771606445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,2,128,1,float16,float16,0,0.5545231819152832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,2,128,1,float16,fp8,0,0.5383999824523926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,2,128,1,fp8,fp8,0,0.5381072044372559
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,4,128,1,float16,float16,0,0.7072591781616211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,4,128,1,float16,fp8,0,0.699945592880249
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,4,128,1,fp8,fp8,0,0.7000912189483642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,1,128,1,float16,float16,0,0.2523008108139038
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,8,128,1,float16,float16,0,0.5286992073059082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,8,128,1,float16,fp8,0,0.5165503978729248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,1,128,1,float16,fp8,0,0.24954240322113036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,8,128,1,fp8,fp8,0,0.5152736186981202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,1,128,1,fp8,fp8,0,0.2504240036010742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,2,128,1,float16,float16,0,0.291761589050293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,2,128,1,float16,fp8,0,0.28821918964385984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,2,128,1,fp8,fp8,0,0.2846208095550537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,4,128,1,float16,float16,0,0.3726815938949585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,8,128,1,fp8,fp8,0,0.27165439128875735
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,4,128,1,float16,fp8,0,0.36620800495147704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,4,128,1,fp8,fp8,0,0.3655904054641724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,8,128,1,float16,float16,0,0.2799583911895752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,8,128,1,float16,fp8,0,0.27163519859313967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,1,128,1,float16,float16,0,0.14234399795532227
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,1,128,1,float16,fp8,0,0.14154399633407594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,1,128,1,fp8,fp8,0,0.1419648051261902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,2,128,1,float16,float16,0,0.16087839603424073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,2,128,1,float16,fp8,0,0.1607136011123657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,2,128,1,fp8,fp8,0,0.15998719930648803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,4,128,1,float16,float16,0,0.2001391887664795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,4,128,1,float16,fp8,0,0.197979199886322
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,4,128,1,fp8,fp8,0,0.19756959676742553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,8,128,1,float16,float16,0,0.1538272023200989
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,8,128,1,float16,fp8,0,0.1515552043914795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,8,128,1,fp8,fp8,0,0.15204800367355348
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,1,128,1,float16,float16,0,0.08595839738845826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,1,128,1,float16,fp8,0,0.08165760040283203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,1,128,1,fp8,fp8,0,0.08276479840278625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,2,128,1,float16,float16,0,0.09448000192642211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,2,128,1,float16,fp8,0,0.0897055983543396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,2,128,1,fp8,fp8,0,0.09212319850921631
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,4,128,1,float16,float16,0,0.11407040357589722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,4,128,1,float16,fp8,0,0.11341919898986816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,4,128,1,fp8,fp8,0,0.1137503981590271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,8,128,1,float16,float16,0,0.09050400257110595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,8,128,1,float16,fp8,0,0.08342080116271973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,8,128,1,fp8,fp8,0,0.08174239993095397
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,1,128,1,float16,float16,0,0.050692802667617796
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,1,128,1,float16,fp8,0,0.052108800411224364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,1,128,1,fp8,fp8,0,0.051755201816558835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,2,128,1,float16,float16,0,0.05696319937705994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,2,128,1,float16,fp8,0,0.05621119737625122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,2,128,1,fp8,fp8,0,0.0562720000743866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,4,128,1,float16,float16,0,0.06660159826278686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,4,128,1,float16,fp8,0,0.06339359879493714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,4,128,1,fp8,fp8,0,0.06375679969787598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,8,128,1,float16,float16,0,0.051420801877975465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,8,128,1,float16,fp8,0,0.05177599787712097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,8,128,1,fp8,fp8,0,0.051849597692489625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,1,128,1,float16,float16,0,0.037862399220466615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,1,128,1,float16,fp8,0,0.039134401082992556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,1,128,1,fp8,fp8,0,0.039340800046920775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,2,128,1,float16,float16,0,0.03884960114955902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,2,128,1,float16,fp8,0,0.04012160003185272
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,2,128,1,fp8,fp8,0,0.04000160098075867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,4,128,1,float16,float16,0,0.04316000044345856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,4,128,1,float16,fp8,0,0.04433920085430145
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,4,128,1,fp8,fp8,0,0.043900799751281736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,8,128,1,float16,float16,0,0.041649600863456725
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,8,128,1,float16,fp8,0,0.04317759871482849
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,8,128,1,fp8,fp8,0,0.043486401438713074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,1,128,1,float16,float16,0,0.03643040060997009
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,1,128,1,float16,fp8,0,0.03776159882545471
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,1,128,1,fp8,fp8,0,0.0377023994922638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,2,128,1,float16,float16,0,0.036267200112342836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,2,128,1,float16,fp8,0,0.03805440068244934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,2,128,1,fp8,fp8,0,0.037950399518013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,4,128,1,float16,float16,0,0.03720000088214874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,4,128,1,float16,fp8,0,0.03901279866695404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,4,128,1,fp8,fp8,0,0.0391728013753891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,1,128,1,float16,float16,0,0.6320735931396484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,1,128,1,float16,fp8,0,0.6319215774536133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,1,128,1,fp8,fp8,0,0.6362080097198486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,2,128,1,float16,float16,0,0.787443208694458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,2,128,1,float16,fp8,0,0.7824927806854248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,2,128,1,fp8,fp8,0,0.7819519996643066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,4,128,1,float16,float16,0,1.1003392219543457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,4,128,1,float16,fp8,0,1.0890288352966309
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,1,128,1,float16,float16,0,0.3290287971496582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,8,128,1,float16,float16,0,0.8805999755859375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,4,128,1,fp8,fp8,0,1.0887727737426758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,8,128,1,float16,fp8,0,0.8633119583129882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,8,128,1,fp8,fp8,0,0.8640543937683105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,1,128,1,float16,fp8,0,0.32968640327453613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,1,128,1,fp8,fp8,0,0.330184006690979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,2,128,1,float16,float16,0,0.4067471981048584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,2,128,1,float16,fp8,0,0.4033872127532959
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,2,128,1,fp8,fp8,0,0.39951679706573484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,4,128,1,float16,float16,0,0.564515209197998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,4,128,1,float16,fp8,0,0.5595759868621826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,8,128,1,float16,float16,0,0.45172319412231443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,4,128,1,fp8,fp8,0,0.5614096164703369
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,8,128,1,float16,fp8,0,0.4421216011047363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,1,128,1,float16,float16,0,0.17674560546875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,8,128,1,fp8,fp8,0,0.4400464057922363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,1,128,1,float16,fp8,0,0.17837439775466918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,1,128,1,fp8,fp8,0,0.17833759784698486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,2,128,1,float16,float16,0,0.21440320014953612
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,2,128,1,float16,fp8,0,0.21358079910278321
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,2,128,1,fp8,fp8,0,0.21563360691070557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,4,128,1,float16,float16,0,0.29163360595703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,4,128,1,float16,fp8,0,0.289574408531189
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,4,128,1,fp8,fp8,0,0.2896431922912598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,8,128,1,float16,float16,0,0.23798880577087403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,8,128,1,float16,fp8,0,0.23220479488372803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,8,128,1,fp8,fp8,0,0.23253920078277587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,1,128,1,float16,float16,0,0.10164480209350586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,1,128,1,float16,fp8,0,0.10219839811325074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,1,128,1,fp8,fp8,0,0.10213439464569092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,4,128,1,fp8,fp8,0,0.15846240520477295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,2,128,1,float16,float16,0,0.11992319822311401
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,2,128,1,float16,fp8,0,0.11980160474777221
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,2,128,1,fp8,fp8,0,0.11987680196762085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,4,128,1,float16,float16,0,0.158460795879364
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,4,128,1,float16,fp8,0,0.15737600326538087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,8,128,1,float16,float16,0,0.13197120428085327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,8,128,1,float16,fp8,0,0.1289199948310852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,8,128,1,fp8,fp8,0,0.12908639907836914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,1,128,1,float16,float16,0,0.06320480108261109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,1,128,1,float16,fp8,0,0.059308797121047974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,1,128,1,fp8,fp8,0,0.05875040292739868
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,2,128,1,float16,float16,0,0.07185760140419006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,2,128,1,float16,fp8,0,0.06857600212097167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,2,128,1,fp8,fp8,0,0.06865760087966918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,4,128,1,float16,float16,0,0.09227359890937806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,4,128,1,float16,fp8,0,0.09094560146331787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,4,128,1,fp8,fp8,0,0.09048320055007934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,8,128,1,float16,float16,0,0.07724480032920837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,8,128,1,float16,fp8,0,0.06961920261383056
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,8,128,1,fp8,fp8,0,0.0687120020389557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,1,128,1,float16,float16,0,0.037462401390075686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,1,128,1,float16,fp8,0,0.03939520120620728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,1,128,1,fp8,fp8,0,0.039504000544548036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,2,128,1,float16,float16,0,0.043105599284172055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,2,128,1,float16,fp8,0,0.043881601095199584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,2,128,1,fp8,fp8,0,0.04366239905357361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,4,128,1,float16,float16,0,0.05281760096549988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,4,128,1,float16,fp8,0,0.05073440074920654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,4,128,1,fp8,fp8,0,0.050753599405288695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,8,128,1,float16,float16,0,0.041886401176452634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,8,128,1,float16,fp8,0,0.04263519942760467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,8,128,1,fp8,fp8,0,0.0424784004688263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,1,128,1,float16,float16,0,0.028921601176261903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,1,128,1,float16,fp8,0,0.030347201228141784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,1,128,1,fp8,fp8,0,0.03009119927883148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,2,128,1,float16,float16,0,0.02941280007362366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,2,128,1,float16,fp8,0,0.030798399448394777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,2,128,1,fp8,fp8,0,0.03070879876613617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,4,128,1,float16,float16,0,0.03377760052680969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,4,128,1,float16,fp8,0,0.035534399747848514
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,4,128,1,fp8,fp8,0,0.03545120060443878
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,8,128,1,float16,float16,0,0.03341279923915863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,8,128,1,float16,fp8,0,0.03500800132751465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,8,128,1,fp8,fp8,0,0.03503519892692566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,1,128,1,float16,float16,0,0.028118398785591126
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,1,128,1,float16,fp8,0,0.02921760082244873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,1,128,1,fp8,fp8,0,0.02996160089969635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,2,128,1,float16,float16,0,0.02805120050907135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,2,128,1,float16,fp8,0,0.02964479923248291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,2,128,1,fp8,fp8,0,0.029516801238059998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,4,128,1,float16,float16,0,0.02879199981689453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,4,128,1,float16,fp8,0,0.03065119981765747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,4,128,1,fp8,fp8,0,0.030687999725341798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,8,128,1,float16,float16,0,0.028585600852966308
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,8,128,1,float16,fp8,0,0.03046880066394806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,8,128,1,fp8,fp8,0,0.03030399978160858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,1,128,1,float16,float16,0,0.027220800518989563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,4,128,1,float16,float16,0,0.027606400847434997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,1,128,1,float16,fp8,0,0.02880159914493561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,1,128,1,fp8,fp8,0,0.029300799965858458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,2,128,1,float16,float16,0,0.027169600129127502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,2,128,1,float16,fp8,0,0.02908959984779358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,2,128,1,fp8,fp8,0,0.02935839891433716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,4,128,1,float16,fp8,0,0.02919520139694214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,4,128,1,fp8,fp8,0,0.029497599601745604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,1,128,1,float16,float16,0,0.2828768014907837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,1,128,1,float16,fp8,0,0.2897488117218018
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,1,128,1,fp8,fp8,0,0.28892319202423095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,2,128,1,float16,float16,0,0.360644793510437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,2,128,1,float16,fp8,0,0.3597743988037109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,2,128,1,fp8,fp8,0,0.3593168020248413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,4,128,1,float16,float16,0,0.5213903903961181
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,4,128,1,float16,fp8,0,0.5163040161132812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,4,128,1,fp8,fp8,0,0.5176655769348144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,8,128,1,float16,float16,0,0.43592000007629395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,1,128,1,float16,float16,0,0.1544144034385681
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,8,128,1,float16,fp8,0,0.4148320198059082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,8,128,1,fp8,fp8,0,0.4147024154663086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,1,128,1,float16,fp8,0,0.15778239965438842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,1,128,1,fp8,fp8,0,0.1574463963508606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,2,128,1,float16,float16,0,0.19374879598617553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,2,128,1,float16,fp8,0,0.19395840167999268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,2,128,1,fp8,fp8,0,0.1927839994430542
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,4,128,1,float16,float16,0,0.27358078956604004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,4,128,1,float16,fp8,0,0.2690144062042236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,4,128,1,fp8,fp8,0,0.26952319145202636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,8,128,1,float16,float16,0,0.2314784049987793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,8,128,1,float16,fp8,0,0.21885759830474855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,8,128,1,fp8,fp8,0,0.21892321109771729
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,1,128,1,float16,float16,0,0.0908191978931427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,1,128,1,float16,fp8,0,0.09176160097122192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,1,128,1,fp8,fp8,0,0.09128320217132568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,4,128,1,fp8,fp8,0,0.1478976011276245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,2,128,1,float16,float16,0,0.10933279991149902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,2,128,1,float16,fp8,0,0.10982400178909302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,2,128,1,fp8,fp8,0,0.10949599742889404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,4,128,1,float16,float16,0,0.14892799854278566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,4,128,1,float16,fp8,0,0.14728000164031982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,8,128,1,float16,float16,0,0.12785279750823975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,8,128,1,float16,fp8,0,0.12076959609985352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,8,128,1,fp8,fp8,0,0.12121920585632324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,1,128,1,float16,float16,0,0.057417601346969604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,1,128,1,float16,fp8,0,0.05488479733467102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,1,128,1,fp8,fp8,0,0.0522383987903595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,2,128,1,float16,float16,0,0.06574400067329407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,2,128,1,float16,fp8,0,0.06257920265197754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,2,128,1,fp8,fp8,0,0.0626479983329773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,4,128,1,float16,float16,0,0.08679680228233337
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,4,128,1,float16,fp8,0,0.08493760228157043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,4,128,1,fp8,fp8,0,0.08377919793128967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,8,128,1,float16,float16,0,0.07282879948616028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,8,128,1,float16,fp8,0,0.06316480040550232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,2,128,1,float16,fp8,0,0.03904159963130951
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,8,128,1,fp8,fp8,0,0.06270080208778381
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,1,128,1,float16,float16,0,0.03238719999790192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,1,128,1,float16,fp8,0,0.0345984011888504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,1,128,1,fp8,fp8,0,0.03442240059375763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,2,128,1,float16,float16,0,0.03825919926166534
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,2,128,1,fp8,fp8,0,0.03903999924659729
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,4,128,1,float16,float16,0,0.047465598583221434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,4,128,1,float16,fp8,0,0.04625760018825531
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,4,128,1,fp8,fp8,0,0.04642080068588257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,8,128,1,float16,float16,0,0.038624000549316403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,8,128,1,float16,fp8,0,0.03866559863090515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,8,128,1,fp8,fp8,0,0.03861759901046753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,1,128,1,float16,float16,0,0.025828799605369566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,1,128,1,float16,fp8,0,0.02645759880542755
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,1,128,1,fp8,fp8,0,0.026368001103401185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,2,128,1,float16,float16,0,0.02635999917984009
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,2,128,1,float16,fp8,0,0.027163198590278624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,2,128,1,fp8,fp8,0,0.027055999636650084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,4,128,1,float16,float16,0,0.030721598863601686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,4,128,1,float16,fp8,0,0.03146080076694489
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,4,128,1,fp8,fp8,0,0.03165439963340759
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,8,128,1,float16,float16,0,0.03033280074596405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,8,128,1,float16,fp8,0,0.03118239939212799
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,8,128,1,fp8,fp8,0,0.031439998745918275
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,1,128,1,float16,float16,0,0.024956800043582916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,1,128,1,float16,fp8,0,0.02566240131855011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,1,128,1,fp8,fp8,0,0.025697600841522217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,2,128,1,float16,float16,0,0.02539519965648651
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,2,128,1,float16,fp8,0,0.0262719988822937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,2,128,1,fp8,fp8,0,0.025887998938560485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,4,128,1,float16,float16,0,0.026107200980186464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,4,128,1,float16,fp8,0,0.02715199887752533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,4,128,1,fp8,fp8,0,0.026791998744010927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,8,128,1,float16,float16,0,0.02592960000038147
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,8,128,1,float16,fp8,0,0.02699199914932251
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,8,128,1,fp8,fp8,0,0.026956799626350402
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,1,128,1,float16,float16,0,0.024641600251197816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,1,128,1,float16,fp8,0,0.025731199979782106
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,1,128,1,fp8,fp8,0,0.02566399872303009
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,2,128,1,float16,float16,0,0.02462719976902008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,2,128,1,float16,fp8,0,0.02544800043106079
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,2,128,1,fp8,fp8,0,0.025824001431465148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,4,128,1,float16,float16,0,0.024801599979400634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,4,128,1,float16,fp8,0,0.02582559883594513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,4,128,1,fp8,fp8,0,0.026281601190567015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,8,128,1,float16,float16,0,0.02324160039424896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,8,128,1,float16,fp8,0,0.02444320023059845
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,8,128,1,fp8,fp8,0,0.024609600007534028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,1,128,1,float16,float16,0,0.023363199830055238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,1,128,1,float16,fp8,0,0.024515199661254882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,4,128,1,float16,fp8,0,0.024553599953651428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,1,128,1,fp8,fp8,0,0.024009600281715393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,2,128,1,float16,float16,0,0.02319519966840744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,2,128,1,float16,fp8,0,0.024265600740909575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,2,128,1,fp8,fp8,0,0.024353599548339842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,4,128,1,float16,float16,0,0.02316479980945587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,4,128,1,fp8,fp8,0,0.024340799450874327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,1,128,1,float16,float16,0,0.15119199752807616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,1,128,1,float16,fp8,0,0.15654720067977906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,1,128,1,fp8,fp8,0,0.15666879415512086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,2,128,1,float16,float16,0,0.19092320203781127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,2,128,1,float16,fp8,0,0.19410879611968995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,2,128,1,fp8,fp8,0,0.19451839923858644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,4,128,1,float16,float16,0,0.2713536024093628
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,4,128,1,float16,fp8,0,0.2705984115600586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,4,128,1,fp8,fp8,0,0.26950559616088865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,8,128,1,float16,float16,0,0.23028960227966308
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,8,128,1,float16,fp8,0,0.21949279308319092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,1,128,1,float16,float16,0,0.08738719820976257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,8,128,1,fp8,fp8,0,0.2180704116821289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,1,128,1,float16,fp8,0,0.08997920155525208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,1,128,1,fp8,fp8,0,0.08959680199623107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,2,128,1,float16,float16,0,0.10618400573730469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,2,128,1,float16,fp8,0,0.10783200263977051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,2,128,1,fp8,fp8,0,0.10733280181884766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,4,128,1,float16,float16,0,0.1456112027168274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,4,128,1,float16,fp8,0,0.144924795627594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,4,128,1,fp8,fp8,0,0.14497920274734497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,8,128,1,float16,float16,0,0.12698880434036255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,8,128,1,float16,fp8,0,0.11858880519866943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,8,128,1,fp8,fp8,0,0.1186527967453003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,1,128,1,float16,float16,0,0.055032002925872806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,1,128,1,float16,fp8,0,0.048651200532913205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,1,128,1,fp8,fp8,0,0.04969600141048432
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,2,128,1,float16,float16,0,0.06391680240631104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,8,128,1,float16,float16,0,0.07092959880828857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,2,128,1,float16,fp8,0,0.058734399080276486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,2,128,1,fp8,fp8,0,0.057980799674987794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,4,128,1,float16,float16,0,0.08479200005531311
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,4,128,1,float16,fp8,0,0.08083519935607911
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,1,128,1,fp8,fp8,0,0.03159359991550446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,4,128,1,fp8,fp8,0,0.08151040077209473
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,8,128,1,float16,fp8,0,0.06018880009651184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,8,128,1,fp8,fp8,0,0.060073602199554446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,1,128,1,float16,float16,0,0.030612799525260925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,1,128,1,float16,fp8,0,0.03153760135173798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,2,128,1,float16,float16,0,0.03532480001449585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,8,128,1,float16,float16,0,0.03711200058460236
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,8,128,1,float16,fp8,0,0.037264001369476316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,2,128,1,float16,fp8,0,0.03580960035324097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,2,128,1,fp8,fp8,0,0.035743999481201175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,4,128,1,float16,float16,0,0.045948800444602964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,4,128,1,float16,fp8,0,0.04269759953022003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,4,128,1,fp8,fp8,0,0.04274399876594544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,2,128,1,fp8,fp8,0,0.02572000026702881
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,8,128,1,fp8,fp8,0,0.0371071994304657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,1,128,1,float16,float16,0,0.02409279942512512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,1,128,1,float16,fp8,0,0.024953599274158477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,1,128,1,fp8,fp8,0,0.024963200092315674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,2,128,1,float16,float16,0,0.025259199738502502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,2,128,1,float16,fp8,0,0.02598400115966797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,4,128,1,float16,float16,0,0.02946079969406128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,4,128,1,float16,fp8,0,0.030031999945640563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,4,128,1,fp8,fp8,0,0.030110400915145875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,8,128,1,float16,float16,0,0.028984001278877257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,8,128,1,float16,fp8,0,0.02961759865283966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,8,128,1,fp8,fp8,0,0.029630398750305174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,1,128,1,float16,float16,0,0.024035200476646423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,1,128,1,float16,fp8,0,0.02433120012283325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,1,128,1,fp8,fp8,0,0.02448800057172775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,2,128,1,float16,float16,0,0.023783999681472778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,2,128,1,float16,fp8,0,0.02433120012283325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,2,128,1,fp8,fp8,0,0.02449920028448105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,4,128,1,float16,float16,0,0.02465600073337555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,4,128,1,float16,fp8,0,0.025348800420761108
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,4,128,1,fp8,fp8,0,0.02550880014896393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,8,128,1,float16,float16,0,0.024396799504756927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,8,128,1,float16,fp8,0,0.02503040134906769
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,8,128,1,fp8,fp8,0,0.024843199551105498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,1,128,1,float16,float16,0,0.02348320037126541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,1,128,1,float16,fp8,0,0.02390560060739517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,1,128,1,fp8,fp8,0,0.024377599358558655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,2,128,1,float16,float16,0,0.02325280010700226
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,2,128,1,float16,fp8,0,0.02388159930706024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,2,128,1,fp8,fp8,0,0.024240000545978545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,4,128,1,float16,float16,0,0.023915199935436247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,4,128,1,float16,fp8,0,0.023963199555873872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,4,128,1,fp8,fp8,0,0.024508799612522125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,8,128,1,float16,float16,0,0.02216159999370575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,8,128,1,float16,fp8,0,0.02268799990415573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,8,128,1,fp8,fp8,0,0.022644799947738648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,1,128,1,float16,float16,0,0.021793599426746368
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,1,128,1,float16,fp8,0,0.022169600427150726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,1,128,1,fp8,fp8,0,0.0221791997551918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,2,128,1,float16,float16,0,0.02178719937801361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,2,128,1,float16,fp8,0,0.02253919988870621
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,2,128,1,fp8,fp8,0,0.02247840017080307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,4,128,1,float16,float16,0,0.021823999285697938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,4,128,1,float16,fp8,0,0.02248319983482361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,4,128,1,fp8,fp8,0,0.022299200296401978
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,8,128,1,float16,float16,0,0.020953600108623505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,8,128,1,float16,fp8,0,0.0217616006731987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,8,128,1,fp8,fp8,0,0.021718400716781616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,1,128,1,float16,float16,0,0.020960000157356263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,1,128,1,float16,fp8,0,0.02152640074491501
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,1,128,1,fp8,fp8,0,0.021572799980640413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,2,128,1,float16,float16,0,0.020870399475097657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,2,128,1,float16,fp8,0,0.02143999934196472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,2,128,1,fp8,fp8,0,0.021547199785709382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,1,128,1,float16,fp8,0,0.090447998046875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,4,128,1,float16,float16,0,0.021009600162506102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,4,128,1,float16,fp8,0,0.021755200624465943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,4,128,1,fp8,fp8,0,0.021631999313831328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,1,128,1,float16,float16,0,0.08915839791297912
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,1,128,1,fp8,fp8,0,0.0906768023967743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,2,128,1,float16,float16,0,0.10858399868011474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,2,128,1,float16,fp8,0,0.1089743971824646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,2,128,1,fp8,fp8,0,0.10951520204544067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,4,128,1,float16,float16,0,0.149510395526886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,4,128,1,float16,fp8,0,0.14649759531021117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,4,128,1,fp8,fp8,0,0.14563679695129395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,8,128,1,float16,float16,0,0.13879519701004028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,8,128,1,float16,fp8,0,0.1369647979736328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,8,128,1,fp8,fp8,0,0.137007999420166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,1,128,1,float16,float16,0,0.0556768000125885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,1,128,1,float16,fp8,0,0.05139039754867554
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,1,128,1,fp8,fp8,0,0.050430399179458615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,2,128,1,float16,float16,0,0.06487039923667907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,2,128,1,float16,fp8,0,0.05889760255813599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,2,128,1,fp8,fp8,0,0.059931200742721555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,4,128,1,float16,float16,0,0.08521119952201843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,4,128,1,float16,fp8,0,0.08169919848442078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,4,128,1,fp8,fp8,0,0.08216000199317933
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,8,128,1,float16,float16,0,0.07910240292549134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,8,128,1,float16,fp8,0,0.06918079853057861
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,8,128,1,fp8,fp8,0,0.06890559792518616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,1,128,1,float16,float16,0,0.030313599109649658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,1,128,1,float16,fp8,0,0.03183839917182922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,1,128,1,fp8,fp8,0,0.03199679851531982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,2,128,1,float16,float16,0,0.035667198896408084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,2,128,1,float16,fp8,0,0.036111998558044436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,2,128,1,fp8,fp8,0,0.03642080128192902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,4,128,1,float16,float16,0,0.04634400010108948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,4,128,1,float16,fp8,0,0.043619200587272644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,4,128,1,fp8,fp8,0,0.043166399002075195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,8,128,1,float16,float16,0,0.041038399934768675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,8,128,1,float16,fp8,0,0.041510400176048276
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,8,128,1,fp8,fp8,0,0.0416047990322113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,1,128,1,float16,float16,0,0.0247871994972229
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,1,128,1,float16,fp8,0,0.025388801097869874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,1,128,1,fp8,fp8,0,0.025303998589515687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,2,128,1,float16,float16,0,0.02560639977455139
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,2,128,1,float16,fp8,0,0.02587679922580719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,2,128,1,fp8,fp8,0,0.026049599051475525
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,4,128,1,float16,float16,0,0.02974880039691925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,4,128,1,float16,fp8,0,0.03028959929943085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,4,128,1,fp8,fp8,0,0.030460798740386964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,8,128,1,float16,float16,0,0.028407999873161317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,8,128,1,float16,fp8,0,0.02964319884777069
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,8,128,1,fp8,fp8,0,0.02961440086364746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,1,128,1,float16,float16,0,0.023604799807071686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,1,128,1,float16,fp8,0,0.024371199309825897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,1,128,1,fp8,fp8,0,0.024291199445724488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,2,128,1,float16,float16,0,0.023532800376415253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,2,128,1,float16,fp8,0,0.024382400512695312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,2,128,1,fp8,fp8,0,0.02456959933042526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,4,128,1,float16,float16,0,0.024694399535655977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,4,128,1,float16,fp8,0,0.025519999861717223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,4,128,1,fp8,fp8,0,0.02510719895362854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,8,128,1,float16,float16,0,0.024031999707221984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,8,128,1,float16,fp8,0,0.02465440034866333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,8,128,1,fp8,fp8,0,0.024886399507522583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,1,128,1,float16,float16,0,0.023183999955654143
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,1,128,1,float16,fp8,0,0.023955200612545014
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,1,128,1,fp8,fp8,0,0.024447999894618988
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,2,128,1,float16,float16,0,0.02322079986333847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,2,128,1,float16,fp8,0,0.023740799725055696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,2,128,1,fp8,fp8,0,0.023681600391864777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,4,128,1,float16,float16,0,0.023296000063419343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,4,128,1,float16,fp8,0,0.024107199907302857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,4,128,1,fp8,fp8,0,0.024129599332809448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,8,128,1,float16,float16,0,0.02165440022945404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,8,128,1,float16,fp8,0,0.02274399995803833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,8,128,1,fp8,fp8,0,0.02245119959115982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,1,128,1,float16,float16,0,0.021456000208854676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,1,128,1,float16,fp8,0,0.02224159985780716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,1,128,1,fp8,fp8,0,0.022204799950122832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,2,128,1,float16,float16,0,0.02178719937801361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,2,128,1,float16,fp8,0,0.02229280024766922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,2,128,1,fp8,fp8,0,0.022622400522232057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,4,128,1,float16,float16,0,0.021753600239753722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,4,128,1,float16,fp8,0,0.022256000339984892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,4,128,1,fp8,fp8,0,0.022329600155353548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,8,128,1,float16,float16,0,0.020921599864959717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,8,128,1,float16,fp8,0,0.021721599996089934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,8,128,1,fp8,fp8,0,0.02160000056028366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,1,128,1,float16,float16,0,0.02094399929046631
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,1,128,1,float16,fp8,0,0.02147040069103241
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,1,128,1,fp8,fp8,0,0.021587200462818146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,2,128,1,float16,float16,0,0.020737600326538087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,2,128,1,float16,fp8,0,0.021489599347114564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,2,128,1,fp8,fp8,0,0.021723200380802155
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,4,128,1,float16,float16,0,0.020817600190639496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,4,128,1,float16,fp8,0,0.021753600239753722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,4,128,1,fp8,fp8,0,0.021473599970340727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,8,128,1,float16,float16,0,0.020446400344371795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,8,128,1,float16,fp8,0,0.021193599700927733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,1,128,1,float16,float16,0,0.02048960030078888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,8,128,1,fp8,fp8,0,0.18519680500030516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,1,128,1,float16,fp8,0,0.021408000588417055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,4,128,1,fp8,fp8,0,0.021457600593566894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,1,128,1,fp8,fp8,0,0.02136639952659607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,2,128,1,float16,float16,0,0.020712000131607056
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,2,128,1,float16,fp8,0,0.021359999477863312
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,2,128,1,fp8,fp8,0,0.021211199462413788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,4,128,1,float16,float16,0,0.020608000457286835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,4,128,1,float16,fp8,0,0.02141599953174591
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,2,128,1,fp8,fp8,0,0.06053599715232849
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,1,128,1,float16,float16,0,0.05669599771499634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,1,128,1,float16,fp8,0,0.05242400169372559
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,1,128,1,fp8,fp8,0,0.05370079874992371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,2,128,1,float16,float16,0,0.066048002243042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,2,128,1,float16,fp8,0,0.06071680188179016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,4,128,1,float16,float16,0,0.10047199726104736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,4,128,1,float16,fp8,0,0.10048320293426513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,4,128,1,fp8,fp8,0,0.10052319765090942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,8,128,1,float16,float16,0,0.09424960017204284
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,8,128,1,float16,fp8,0,0.08716639876365662
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,8,128,1,fp8,fp8,0,0.08749759793281556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,1,128,1,float16,float16,0,0.031472000479698184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,1,128,1,float16,fp8,0,0.03281440138816834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,1,128,1,fp8,fp8,0,0.0331712007522583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,2,128,1,float16,float16,0,0.036078399419784545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,2,128,1,float16,fp8,0,0.03734720051288605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,2,128,1,fp8,fp8,0,0.037303999066352844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,4,128,1,float16,float16,0,0.05307040214538574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,4,128,1,float16,fp8,0,0.052616000175476074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,4,128,1,fp8,fp8,0,0.053406399488449094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,8,128,1,float16,float16,0,0.050228798389434816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,8,128,1,float16,fp8,0,0.05106559991836548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,8,128,1,fp8,fp8,0,0.050695997476577756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,1,128,1,float16,float16,0,0.02521440088748932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,1,128,1,float16,fp8,0,0.025966399908065797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,1,128,1,fp8,fp8,0,0.02600319981575012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,2,128,1,float16,float16,0,0.02587679922580719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,2,128,1,float16,fp8,0,0.026833599805831908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,2,128,1,fp8,fp8,0,0.026848000288009644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,4,128,1,float16,float16,0,0.03348959982395172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,4,128,1,float16,fp8,0,0.034939199686050415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,4,128,1,fp8,fp8,0,0.03499839901924133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,8,128,1,float16,float16,0,0.032691198587417605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,8,128,1,float16,fp8,0,0.033964800834655764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,8,128,1,fp8,fp8,0,0.03416160047054291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,1,128,1,float16,float16,0,0.024247999489307403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,1,128,1,float16,fp8,0,0.024952000379562377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,1,128,1,fp8,fp8,0,0.024747200310230255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,2,128,1,float16,float16,0,0.024260799586772918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,2,128,1,float16,fp8,0,0.02460159957408905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,2,128,1,fp8,fp8,0,0.02476480007171631
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,4,128,1,float16,float16,0,0.02463040053844452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,4,128,1,float16,fp8,0,0.02534559965133667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,4,128,1,fp8,fp8,0,0.02571519911289215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,8,128,1,float16,float16,0,0.024160000681877136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,8,128,1,float16,fp8,0,0.025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,8,128,1,fp8,fp8,0,0.02496960014104843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,1,128,1,float16,float16,0,0.022974400222301482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,1,128,1,float16,fp8,0,0.02415039986371994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,1,128,1,fp8,fp8,0,0.02425439953804016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,2,128,1,float16,float16,0,0.023532800376415253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,2,128,1,float16,fp8,0,0.023924799263477327
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,2,128,1,fp8,fp8,0,0.024128000438213348
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,4,128,1,float16,float16,0,0.023132799565792082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,4,128,1,float16,fp8,0,0.02434239983558655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,4,128,1,fp8,fp8,0,0.02385119944810867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,8,128,1,float16,float16,0,0.021873599290847777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,8,128,1,float16,fp8,0,0.022672000527381896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,8,128,1,fp8,fp8,0,0.02294880002737045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,1,128,1,float16,float16,0,0.021932800114154816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,1,128,1,float16,fp8,0,0.022672000527381896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,1,128,1,fp8,fp8,0,0.022542400658130644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,2,128,1,float16,float16,0,0.0217616006731987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,2,128,1,float16,fp8,0,0.02261119931936264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,2,128,1,fp8,fp8,0,0.022489599883556366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,4,128,1,float16,float16,0,0.022049599885940553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,4,128,1,float16,fp8,0,0.022742399573326112
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,4,128,1,fp8,fp8,0,0.022364799678325654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,8,128,1,float16,float16,0,0.02093919962644577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,8,128,1,float16,fp8,0,0.021774399280548095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,8,128,1,fp8,fp8,0,0.021806399524211883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,1,128,1,float16,float16,0,0.020665599405765532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,1,128,1,float16,fp8,0,0.021704000234603883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,1,128,1,fp8,fp8,0,0.021598400175571443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,2,128,1,float16,float16,0,0.020766399800777435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,2,128,1,float16,fp8,0,0.02146719992160797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,2,128,1,fp8,fp8,0,0.021486400067806243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,4,128,1,float16,float16,0,0.020919999480247496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,4,128,1,float16,fp8,0,0.02170239984989166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,4,128,1,fp8,fp8,0,0.021686400473117828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,8,128,1,float16,float16,0,0.02064639925956726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,8,128,1,float16,fp8,0,0.021326400339603424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,8,128,1,fp8,fp8,0,0.02147199958562851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,1,128,1,float16,float16,0,0.020257599651813507
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,1,128,1,float16,fp8,0,0.021454399824142455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,1,128,1,fp8,fp8,0,0.021132799983024596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,2,128,1,float16,float16,0,0.020681600272655486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,2,128,1,float16,fp8,0,0.0213919997215271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,2,128,1,fp8,fp8,0,0.02127680033445358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,4,128,1,float16,float16,0,0.020411199331283568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,4,128,1,float16,fp8,0,0.021052800118923187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,4,128,1,fp8,fp8,0,0.021238400042057036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,8,128,1,float16,float16,0,0.02046079933643341
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,8,128,1,float16,fp8,0,0.02105119973421097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,8,128,1,fp8,fp8,0,0.021236799657344818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,1,128,1,float16,float16,0,0.0204927995800972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,1,128,1,float16,fp8,0,0.021278400719165803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,1,128,1,fp8,fp8,0,0.02144320011138916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,2,128,1,float16,float16,0,0.020351999998092653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,2,128,1,float16,fp8,0,0.021169599890708924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,2,128,1,fp8,fp8,0,0.021227200329303742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,4,128,1,float16,float16,0,0.02019519954919815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,4,128,1,float16,fp8,0,0.021331200003623964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,4,128,1,fp8,fp8,0,0.0211776003241539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,1,128,1,float16,float16,0,0.033939200639724734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,1,128,1,float16,fp8,0,0.03506399989128113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,1,128,1,fp8,fp8,0,0.03541440069675446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,2,128,1,float16,float16,0,0.04486719965934753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,2,128,1,float16,fp8,0,0.047937598824501035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,2,128,1,fp8,fp8,0,0.04771359860897064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,4,128,1,float16,float16,0,0.06998559832572937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,4,128,1,float16,fp8,0,0.07268480062484742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,4,128,1,fp8,fp8,0,0.07279999852180481
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,8,128,1,float16,float16,0,0.06471040248870849
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,8,128,1,float16,fp8,0,0.06971200108528137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,8,128,1,fp8,fp8,0,0.06999359726905822
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,1,128,1,float16,float16,0,0.026723200082778932
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,1,128,1,float16,fp8,0,0.027030399441719054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,1,128,1,fp8,fp8,0,0.02720640003681183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,2,128,1,float16,float16,0,0.03040800094604492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,2,128,1,float16,fp8,0,0.032393598556518556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,2,128,1,fp8,fp8,0,0.03193599879741669
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,4,128,1,float16,float16,0,0.04205600023269653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,4,128,1,float16,fp8,0,0.045368000864982605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,4,128,1,fp8,fp8,0,0.04532000124454498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,8,128,1,float16,float16,0,0.040673598647117615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,8,128,1,float16,fp8,0,0.043932801485061644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,8,128,1,fp8,fp8,0,0.04380480051040649
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,1,128,1,float16,float16,0,0.02441119998693466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,1,128,1,float16,fp8,0,0.025516799092292784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,1,128,1,fp8,fp8,0,0.025016000866889952
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,2,128,1,float16,float16,0,0.024481600522994994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,2,128,1,float16,fp8,0,0.02526240050792694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,2,128,1,fp8,fp8,0,0.025649601221084596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,4,128,1,float16,float16,0,0.029359999299049377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,4,128,1,float16,fp8,0,0.030608001351356506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,4,128,1,fp8,fp8,0,0.030640000104904176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,8,128,1,float16,float16,0,0.028270399570465087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,8,128,1,float16,fp8,0,0.02958880066871643
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,8,128,1,fp8,fp8,0,0.029788801074028017
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,1,128,1,float16,float16,0,0.023688000440597535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,1,128,1,float16,fp8,0,0.02481119930744171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,1,128,1,fp8,fp8,0,0.025252801179885865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,2,128,1,float16,float16,0,0.023505599796772005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,2,128,1,float16,fp8,0,0.02422720044851303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,2,128,1,fp8,fp8,0,0.02439360022544861
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,4,128,1,float16,float16,0,0.023707200586795808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,4,128,1,float16,fp8,0,0.024607999622821806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,4,128,1,fp8,fp8,0,0.02462400048971176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,8,128,1,float16,float16,0,0.022350400686264038
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,8,128,1,float16,fp8,0,0.023105600476264955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,8,128,1,fp8,fp8,0,0.023028799891471864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,1,128,1,float16,float16,0,0.021729600429534913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,1,128,1,float16,fp8,0,0.022350400686264038
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,1,128,1,fp8,fp8,0,0.02237759977579117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,2,128,1,float16,float16,0,0.021899199485778807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,2,128,1,float16,fp8,0,0.02245119959115982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,8,128,1,float16,fp8,0,0.022300800681114195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,2,128,1,fp8,fp8,0,0.022873599827289582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,1,128,1,float16,float16,0,0.020942400395870208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,4,128,1,float16,float16,0,0.021862399578094483
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,4,128,1,float16,fp8,0,0.022515200078487396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,4,128,1,fp8,fp8,0,0.022646400332450866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,8,128,1,float16,float16,0,0.02105119973421097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,8,128,1,fp8,fp8,0,0.022094400227069856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,1,128,1,float16,fp8,0,0.021612800657749176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,1,128,1,fp8,fp8,0,0.021876800060272216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,2,128,1,float16,float16,0,0.020839999616146087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,2,128,1,float16,fp8,0,0.021852800250053407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,2,128,1,fp8,fp8,0,0.021652799844741822
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,4,128,1,float16,float16,0,0.020883199572563172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,4,128,1,float16,fp8,0,0.022086399793624877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,4,128,1,fp8,fp8,0,0.02176000028848648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,8,128,1,float16,float16,0,0.021054400503635405
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,8,128,1,float16,fp8,0,0.021649600565433504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,8,128,1,fp8,fp8,0,0.021745599806308746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,1,128,1,float16,float16,0,0.020684799551963805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,1,128,1,float16,fp8,0,0.021631999313831328
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,1,128,1,fp8,fp8,0,0.021329599618911742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,2,128,1,float16,float16,0,0.020295999944210052
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,2,128,1,float16,fp8,0,0.021220800280570985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,2,128,1,fp8,fp8,0,0.021201600134372712
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,4,128,1,float16,float16,0,0.020443199574947356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,4,128,1,float16,fp8,0,0.02130880057811737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,4,128,1,fp8,fp8,0,0.020972800254821778
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,8,128,1,float16,float16,0,0.020521600544452668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,8,128,1,float16,fp8,0,0.021249599754810333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,8,128,1,fp8,fp8,0,0.021432000398635864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,1,128,1,float16,float16,0,0.020468799769878386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,1,128,1,float16,fp8,0,0.021406400203704833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,1,128,1,fp8,fp8,0,0.021334399282932282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,2,128,1,float16,float16,0,0.020177599787712098
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,2,128,1,float16,fp8,0,0.021033599972724915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,2,128,1,fp8,fp8,0,0.02109760046005249
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,4,128,1,float16,float16,0,0.020364800095558168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,4,128,1,float16,fp8,0,0.021160000562667848
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,4,128,1,fp8,fp8,0,0.021155199408531188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,8,128,1,float16,float16,0,0.020268799364566804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,8,128,1,float16,fp8,0,0.02107519954442978
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,8,128,1,fp8,fp8,0,0.02102559953927994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,1,128,1,float16,float16,0,0.020206399261951447
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,1,128,1,float16,fp8,0,0.021329599618911742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,1,128,1,fp8,fp8,0,0.021270400285720824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,2,128,1,float16,float16,0,0.02009280025959015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,2,128,1,float16,fp8,0,0.021134400367736818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,2,128,1,fp8,fp8,0,0.021111999452114106
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,4,128,1,float16,float16,0,0.02019679993391037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,4,128,1,float16,fp8,0,0.02085919976234436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,4,128,1,fp8,fp8,0,0.020979200303554536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,4,128,1,float16,fp8,0,1.886128044128418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,4,128,1,float16,float16,0,2.065283203125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,4,1,128,1,float16,fp8,0,3.2783695220947267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,4,1,128,1,float16,float16,0,3.6562545776367186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,4,1,128,1,fp8,fp8,0,3.2503902435302736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,4,2,128,1,float16,fp8,0,3.4061649322509764
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,4,2,128,1,float16,float16,0,3.9297840118408205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,4,2,128,1,fp8,fp8,0,3.3916481018066404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,4,128,1,fp8,fp8,0,1.878447914123535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,1,128,1,float16,fp8,0,1.6504880905151367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,1,128,1,float16,float16,0,1.8639728546142578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,1,128,1,fp8,fp8,0,1.7531440734863282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,4,128,1,float16,float16,0,1.0047871589660644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,2,128,1,float16,float16,0,1.891164779663086
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,2,128,1,float16,fp8,0,1.7149791717529297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,2,128,1,fp8,fp8,0,1.7330720901489258
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,4,128,1,float16,fp8,0,0.9570783615112305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,4,128,1,fp8,fp8,0,0.9532496452331543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,1,128,1,float16,float16,0,0.9238032341003418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,1,128,1,float16,fp8,0,0.8634127616882324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,1,128,1,fp8,fp8,0,0.8640624046325683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,4,128,1,float16,float16,0,0.5178271770477295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,2,128,1,float16,float16,0,0.9802687644958497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,2,128,1,float16,fp8,0,0.8754416465759277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,4,128,1,float16,fp8,0,0.49292478561401365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,2,128,1,fp8,fp8,0,0.8727215766906739
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,1,128,1,float16,float16,0,0.45538721084594724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,4,128,1,fp8,fp8,0,0.5430880069732666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,1,128,1,float16,fp8,0,0.4322256088256836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,1,128,1,fp8,fp8,0,0.43933758735656736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,2,128,1,float16,float16,0,0.5019872188568115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,2,128,1,float16,fp8,0,0.4536896228790283
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,2,128,1,fp8,fp8,0,0.459446382522583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,4,1,128,1,float16,float16,0,2.068452835083008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,4,1,128,1,float16,fp8,0,1.928335952758789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,4,128,1,float16,float16,0,1.2692272186279296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,4,1,128,1,fp8,fp8,0,1.9725776672363282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,4,128,1,float16,fp8,0,1.1625344276428222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,4,2,128,1,float16,float16,0,2.190764808654785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,4,2,128,1,float16,fp8,0,2.023569679260254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,4,2,128,1,fp8,fp8,0,2.0452287673950194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,1,128,1,float16,float16,0,1.0308671951293946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,4,128,1,fp8,fp8,0,1.2993295669555665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,1,128,1,float16,fp8,0,0.9800335884094238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,1,128,1,fp8,fp8,0,0.9915807723999024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,4,128,1,float16,float16,0,0.6642240047454834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,2,128,1,float16,float16,0,1.1060640335083007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,2,128,1,float16,fp8,0,1.130897617340088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,2,128,1,fp8,fp8,0,1.0199263572692872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,4,128,1,float16,fp8,0,0.6240848064422607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,1,128,1,float16,float16,0,0.5308095932006835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,4,128,1,fp8,fp8,0,0.614359998703003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,1,128,1,float16,fp8,0,0.5101535797119141
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,1,128,1,fp8,fp8,0,0.5523200035095215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,2,128,1,float16,float16,0,0.5912144184112549
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,2,128,1,float16,fp8,0,0.5250624179840088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,2,128,1,fp8,fp8,0,0.5434415817260743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,4,128,1,float16,float16,0,0.3684783935546875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,4,128,1,float16,fp8,0,0.3271647930145264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,4,128,1,fp8,fp8,0,0.3263535976409912
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,1,128,1,float16,float16,0,0.30982398986816406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,1,128,1,float16,fp8,0,0.27455360889434816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,1,128,1,fp8,fp8,0,0.26689920425415037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,2,128,1,float16,float16,0,0.310152006149292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,2,128,1,float16,fp8,0,0.2889408111572266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,2,128,1,fp8,fp8,0,0.28877279758453367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,4,1,128,1,float16,float16,0,1.452409553527832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,4,1,128,1,float16,fp8,0,1.3878640174865722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,4,1,128,1,fp8,fp8,0,1.3858320236206054
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,4,128,1,float16,float16,0,0.9375056266784668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,4,2,128,1,float16,float16,0,1.5849679946899413
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,4,2,128,1,float16,fp8,0,1.4679295539855957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,4,2,128,1,fp8,fp8,0,1.4713871955871582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,4,128,1,float16,fp8,0,0.8705007553100585
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,1,128,1,float16,float16,0,0.7378128051757813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,4,128,1,fp8,fp8,0,0.8690239906311035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,1,128,1,float16,fp8,0,0.7160255908966064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,1,128,1,fp8,fp8,0,0.7236608028411865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,2,128,1,float16,float16,0,0.8089695930480957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,2,128,1,float16,fp8,0,0.7544112205505371
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,4,128,1,float16,float16,0,0.5009712219238281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,2,128,1,fp8,fp8,0,0.7580351829528809
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,4,128,1,float16,fp8,0,0.4581888198852539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,4,128,1,fp8,fp8,0,0.4587696075439453
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,1,128,1,float16,float16,0,0.38980960845947266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,1,128,1,float16,fp8,0,0.37406721115112307
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,1,128,1,fp8,fp8,0,0.3762943983078003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,2,128,1,float16,float16,0,0.423360013961792
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,2,128,1,float16,fp8,0,0.3940576076507568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,2,128,1,fp8,fp8,0,0.39477760791778566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,4,128,1,float16,float16,0,0.2925008058547974
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,4,128,1,float16,fp8,0,0.24216799736022948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,4,128,1,fp8,fp8,0,0.24043519496917726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,1,128,1,float16,float16,0,0.21383359432220458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,1,128,1,float16,fp8,0,0.199454402923584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,1,128,1,fp8,fp8,0,0.20095200538635255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,2,128,1,float16,float16,0,0.2265984058380127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,2,128,1,float16,fp8,0,0.2113248109817505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,2,128,1,fp8,fp8,0,0.2121583938598633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,4,1,128,1,float16,float16,0,1.954515266418457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,4,1,128,1,float16,fp8,0,1.800379180908203
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,4,1,128,1,fp8,fp8,0,1.8023296356201173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,4,128,1,float16,float16,0,1.203985595703125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,4,128,1,float16,fp8,0,1.1536800384521484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,4,2,128,1,float16,float16,0,2.1341808319091795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,4,2,128,1,float16,fp8,0,1.954020881652832
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,4,2,128,1,fp8,fp8,0,1.9346063613891602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,4,128,1,fp8,fp8,0,1.2053791999816894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,1,128,1,float16,float16,0,0.9740048408508301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,1,128,1,float16,fp8,0,0.9165136337280273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,1,128,1,fp8,fp8,0,0.9167039871215821
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,2,128,1,float16,float16,0,1.0469951629638672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,4,128,1,float16,float16,0,0.6120384216308594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,2,128,1,float16,fp8,0,1.0997247695922852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,2,128,1,fp8,fp8,0,0.9872256278991699
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,4,128,1,float16,fp8,0,0.6111167907714844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,4,128,1,fp8,fp8,0,0.6083104133605957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,1,128,1,float16,float16,0,0.49491519927978517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,1,128,1,float16,fp8,0,0.4738895893096924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,1,128,1,fp8,fp8,0,0.4998879909515381
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,2,128,1,float16,float16,0,0.5339935779571533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,4,128,1,float16,float16,0,0.3223567962646484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,2,128,1,float16,fp8,0,0.5232079982757568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,2,128,1,fp8,fp8,0,0.5046864032745362
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,4,128,1,float16,fp8,0,0.30895519256591797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,4,128,1,fp8,fp8,0,0.3063055992126465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,1,128,1,float16,float16,0,0.25778560638427733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,1,128,1,float16,fp8,0,0.2592576026916504
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,1,128,1,fp8,fp8,0,0.24487841129302979
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,2,128,1,float16,float16,0,0.28056800365448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,2,128,1,float16,fp8,0,0.2743024110794067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,2,128,1,fp8,fp8,0,0.26218080520629883
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,4,128,1,float16,float16,0,0.17090719938278198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,4,128,1,float16,fp8,0,0.15832159519195557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,4,128,1,fp8,fp8,0,0.15739519596099855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,1,128,1,float16,float16,0,0.1353503942489624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,1,128,1,float16,fp8,0,0.12791839838027955
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,1,128,1,fp8,fp8,0,0.12992479801177978
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,2,128,1,float16,float16,0,0.1486688017845154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,2,128,1,float16,fp8,0,0.13401440382003785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,2,128,1,fp8,fp8,0,0.137881600856781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,4,1,128,1,float16,float16,0,1.1370320320129395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,4,1,128,1,float16,fp8,0,1.075220775604248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,4,1,128,1,fp8,fp8,0,1.0881008148193358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,4,2,128,1,float16,float16,0,1.2537280082702638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,4,2,128,1,float16,fp8,0,1.198134422302246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,4,128,1,float16,float16,0,0.7719935894012451
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,4,2,128,1,fp8,fp8,0,1.1963839530944824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,4,128,1,float16,fp8,0,0.7280223846435547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,4,128,1,fp8,fp8,0,0.7244239807128906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,1,128,1,float16,float16,0,0.5769152164459228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,1,128,1,float16,fp8,0,0.5557807922363281
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,1,128,1,fp8,fp8,0,0.5562416076660156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,2,128,1,float16,float16,0,0.6348432064056396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,2,128,1,float16,fp8,0,0.6156864166259766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,2,128,1,fp8,fp8,0,0.6143519878387451
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,4,128,1,float16,float16,0,0.4119408130645752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,1,128,1,float16,fp8,0,0.2993119955062866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,4,128,1,float16,fp8,0,0.37571680545806885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,4,128,1,fp8,fp8,0,0.37902719974517823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,1,128,1,float16,float16,0,0.30674240589141843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,1,128,1,fp8,fp8,0,0.29984800815582274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,2,128,1,float16,float16,0,0.3377824068069458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,2,128,1,float16,fp8,0,0.32420639991760253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,2,128,1,fp8,fp8,0,0.3241967916488647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,4,128,1,float16,float16,0,0.22527039051055908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,4,128,1,float16,fp8,0,0.20436320304870606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,4,128,1,fp8,fp8,0,0.20368640422821044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,1,128,1,float16,float16,0,0.17293920516967773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,1,128,1,float16,fp8,0,0.1577455997467041
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,1,128,1,fp8,fp8,0,0.15875359773635864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,2,128,1,float16,float16,0,0.18638399839401246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,2,128,1,float16,fp8,0,0.17348799705505372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,2,128,1,fp8,fp8,0,0.17176320552825927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,4,128,1,float16,float16,0,0.12861280441284179
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,4,128,1,float16,fp8,0,0.11850559711456299
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,4,128,1,fp8,fp8,0,0.11832000017166137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,1,128,1,float16,float16,0,0.10159840583801269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,1,128,1,float16,fp8,0,0.10084480047225952
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,1,128,1,fp8,fp8,0,0.10083999633789062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,2,128,1,float16,float16,0,0.10931680202484131
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,2,128,1,float16,fp8,0,0.10727039575576783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,2,128,1,fp8,fp8,0,0.10681439638137817
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,4,1,128,1,float16,float16,0,1.1051119804382323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,4,1,128,1,float16,fp8,0,1.0504511833190917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,4,1,128,1,fp8,fp8,0,1.0526639938354492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,4,2,128,1,float16,float16,0,1.2622480392456055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,4,128,1,float16,float16,0,0.8080096244812012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,4,2,128,1,float16,fp8,0,1.1951135635375976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,4,2,128,1,fp8,fp8,0,1.198977565765381
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,1,128,1,float16,float16,0,0.5623951911926269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,4,128,1,float16,fp8,0,0.7662320137023926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,1,128,1,float16,fp8,0,0.5410048007965088
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,4,128,1,fp8,fp8,0,0.7648863792419434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,1,128,1,fp8,fp8,0,0.5512671947479248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,2,128,1,float16,float16,0,0.6469632148742676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,2,128,1,float16,fp8,0,0.6053423881530762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,4,128,1,float16,float16,0,0.4157584190368652
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,1,128,1,float16,fp8,0,0.28803040981292727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,2,128,1,fp8,fp8,0,0.6103007793426514
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,4,128,1,float16,fp8,0,0.394979190826416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,4,128,1,fp8,fp8,0,0.3952303886413574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,1,128,1,float16,float16,0,0.28983519077301023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,1,128,1,fp8,fp8,0,0.28550560474395753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,2,128,1,float16,float16,0,0.330678391456604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,2,128,1,float16,fp8,0,0.32187039852142335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,1,128,1,float16,fp8,0,0.14986720085144042
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,2,128,1,fp8,fp8,0,0.3220911979675293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,4,128,1,float16,float16,0,0.21915359497070314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,4,128,1,float16,fp8,0,0.20961918830871581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,4,128,1,fp8,fp8,0,0.2095855951309204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,1,128,1,float16,float16,0,0.15668799877166747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,1,128,1,fp8,fp8,0,0.14771519899368285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,4,128,1,fp8,fp8,0,0.10844960212707519
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,2,128,1,float16,float16,0,0.18012319803237914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,2,128,1,float16,fp8,0,0.17238240242004393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,2,128,1,fp8,fp8,0,0.17285920381546022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,4,128,1,float16,float16,0,0.11858400106430053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,4,128,1,float16,fp8,0,0.10740000009536743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,1,128,1,float16,float16,0,0.08202880024909973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,4,128,1,float16,float16,0,0.08695679903030396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,1,128,1,float16,fp8,0,0.08109440207481385
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,1,128,1,fp8,fp8,0,0.08218240141868591
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,2,128,1,float16,float16,0,0.09572799801826477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,2,128,1,float16,fp8,0,0.08905439972877502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,2,128,1,fp8,fp8,0,0.08979679942131043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,4,128,1,float16,fp8,0,0.08404160141944886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,4,128,1,fp8,fp8,0,0.08271679878234864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,1,128,1,float16,float16,0,0.07343999743461609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,1,128,1,float16,fp8,0,0.07197920083999634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,1,128,1,fp8,fp8,0,0.07189279794692993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,2,128,1,float16,float16,0,0.07769759893417358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,2,128,1,float16,fp8,0,0.07680799961090087
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,2,128,1,fp8,fp8,0,0.07893440127372742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,4,1,128,1,float16,float16,0,0.7317008018493653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,4,1,128,1,float16,fp8,0,0.7001664161682128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,4,1,128,1,fp8,fp8,0,0.7029920101165772
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,4,2,128,1,float16,float16,0,0.8474176406860352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,4,2,128,1,float16,fp8,0,0.809331226348877
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,4,128,1,float16,float16,0,0.5637792110443115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,1,128,1,float16,float16,0,0.38431360721588137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,4,2,128,1,fp8,fp8,0,0.8134464263916016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,4,128,1,float16,fp8,0,0.5428048133850097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,4,128,1,fp8,fp8,0,0.5403567790985108
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,1,128,1,float16,fp8,0,0.3715759992599487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,1,128,1,fp8,fp8,0,0.36776158809661863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,2,128,1,float16,float16,0,0.4383855819702148
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,2,128,1,float16,fp8,0,0.42472162246704104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,2,128,1,fp8,fp8,0,0.42525439262390136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,4,128,1,float16,float16,0,0.2995232105255127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,4,128,1,float16,fp8,0,0.28580799102783205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,4,128,1,fp8,fp8,0,0.28387200832366943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,2,128,1,float16,float16,0,0.2380143880844116
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,1,128,1,float16,float16,0,0.20814719200134277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,1,128,1,float16,fp8,0,0.2020143985748291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,1,128,1,fp8,fp8,0,0.20290400981903076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,2,128,1,float16,fp8,0,0.23057279586791993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,2,128,1,fp8,fp8,0,0.23005599975585939
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,4,128,1,float16,float16,0,0.16265439987182617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,4,128,1,float16,fp8,0,0.15819679498672484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,4,128,1,fp8,fp8,0,0.157315194606781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,1,128,1,float16,float16,0,0.11624799966812134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,1,128,1,float16,fp8,0,0.11091040372848511
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,1,128,1,fp8,fp8,0,0.11143840551376342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,2,128,1,float16,float16,0,0.13359040021896362
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,2,128,1,float16,fp8,0,0.12482240200042724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,2,128,1,fp8,fp8,0,0.12543679475784303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,4,128,1,float16,float16,0,0.09260960221290589
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,4,128,1,float16,fp8,0,0.08570240139961242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,4,128,1,fp8,fp8,0,0.08590239882469178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,1,128,1,float16,float16,0,0.06682400107383728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,1,128,1,float16,fp8,0,0.06790879964828492
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,1,128,1,fp8,fp8,0,0.06776800155639648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,2,128,1,float16,float16,0,0.07435839772224426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,2,128,1,float16,fp8,0,0.07388960123062134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,2,128,1,fp8,fp8,0,0.07449600100517273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,1,128,1,float16,fp8,0,0.05292159914970398
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,4,128,1,float16,float16,0,0.0740559995174408
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,4,128,1,float16,fp8,0,0.07270560264587403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,4,128,1,fp8,fp8,0,0.0730463981628418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,1,128,1,float16,float16,0,0.053939199447631835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,1,128,1,fp8,fp8,0,0.054721599817276
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,2,128,1,float16,float16,0,0.06273120045661926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,2,128,1,float16,fp8,0,0.062249600887298584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,2,128,1,fp8,fp8,0,0.06215839982032776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,4,1,128,1,float16,float16,0,0.7620944023132324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,4,1,128,1,float16,fp8,0,0.7263535976409912
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,4,1,128,1,fp8,fp8,0,0.7310160160064697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,4,2,128,1,float16,fp8,0,0.8807904243469238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,4,2,128,1,float16,float16,0,0.9131104469299316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,4,128,1,float16,float16,0,0.6332240104675293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,4,2,128,1,fp8,fp8,0,0.8847824096679687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,4,128,1,float16,fp8,0,0.6110688209533691
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,1,128,1,fp8,fp8,0,0.38023359775543214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,4,128,1,fp8,fp8,0,0.614081621170044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,1,128,1,float16,float16,0,0.3933680057525635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,1,128,1,float16,fp8,0,0.38500161170959474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,2,128,1,float16,float16,0,0.4736767768859863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,2,128,1,float16,fp8,0,0.4566336154937744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,2,128,1,fp8,fp8,0,0.45981760025024415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,4,128,1,float16,float16,0,0.3357215881347656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,4,128,1,float16,fp8,0,0.32102079391479493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,4,128,1,fp8,fp8,0,0.32377920150756834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,1,128,1,float16,float16,0,0.21515519618988038
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,1,128,1,float16,fp8,0,0.20915679931640624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,1,128,1,fp8,fp8,0,0.20954720973968505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,2,128,1,float16,float16,0,0.25391039848327634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,2,128,1,float16,fp8,0,0.2482448101043701
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,2,128,1,fp8,fp8,0,0.24723680019378663
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,4,128,1,float16,float16,0,0.18490240573883057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,4,128,1,float16,fp8,0,0.17996959686279296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,4,128,1,fp8,fp8,0,0.18078559637069702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,1,128,1,float16,float16,0,0.1242143988609314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,1,128,1,float16,fp8,0,0.12077120542526246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,1,128,1,fp8,fp8,0,0.11853920221328736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,2,128,1,float16,float16,0,0.14459680318832396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,2,128,1,float16,fp8,0,0.1411631941795349
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,2,128,1,fp8,fp8,0,0.14116480350494384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,4,128,1,float16,float16,0,0.10856159925460815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,4,128,1,float16,fp8,0,0.10023679733276367
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,4,128,1,fp8,fp8,0,0.09751520156860352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,1,128,1,float16,float16,0,0.07394400238990784
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,1,128,1,float16,fp8,0,0.07271360158920288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,1,128,1,fp8,fp8,0,0.07267199754714966
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,2,128,1,float16,float16,0,0.08460320234298706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,2,128,1,float16,fp8,0,0.08139200210571289
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,2,128,1,fp8,fp8,0,0.08060799837112427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,4,128,1,float16,float16,0,0.06411839723587036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,4,128,1,float16,fp8,0,0.06291519999504089
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,4,128,1,fp8,fp8,0,0.06365119814872741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,1,128,1,float16,float16,0,0.0512224018573761
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,1,128,1,float16,fp8,0,0.051316797733306885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,1,128,1,fp8,fp8,0,0.05151680111885071
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,2,128,1,float16,float16,0,0.05611039996147156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,2,128,1,float16,fp8,0,0.05610560178756714
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,2,128,1,fp8,fp8,0,0.05641440153121948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,4,128,1,float16,float16,0,0.04769600033760071
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,4,128,1,float16,fp8,0,0.0491104006767273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,4,128,1,fp8,fp8,0,0.04857600033283234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,1,128,1,float16,float16,0,0.04248799979686737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,1,128,1,float16,fp8,0,0.043742400407791135
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,1,128,1,fp8,fp8,0,0.04386880099773407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,2,128,1,float16,float16,0,0.04366079866886139
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,2,128,1,float16,fp8,0,0.044791999459266665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,2,128,1,fp8,fp8,0,0.044180798530578616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,4,1,128,1,float16,float16,0,0.5033232212066651
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,4,1,128,1,float16,fp8,0,0.4829103946685791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,4,1,128,1,fp8,fp8,0,0.4816112041473389
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,4,2,128,1,float16,float16,0,0.6179759979248047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,4,2,128,1,float16,fp8,0,0.6017024040222168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,4,128,1,float16,float16,0,0.4456575870513916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,4,2,128,1,fp8,fp8,0,0.6042575836181641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,4,128,1,float16,fp8,0,0.4315360069274902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,4,128,1,fp8,fp8,0,0.42917118072509763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,1,128,1,float16,float16,0,0.26727519035339353
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,1,128,1,float16,fp8,0,0.26166720390319825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,1,128,1,fp8,fp8,0,0.2610960006713867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,2,128,1,float16,float16,0,0.32692000865936277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,2,128,1,float16,fp8,0,0.31863999366760254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,2,128,1,fp8,fp8,0,0.3173743963241577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,4,128,1,float16,float16,0,0.23825759887695314
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,4,128,1,float16,fp8,0,0.230295991897583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,4,128,1,fp8,fp8,0,0.23083360195159913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,1,128,1,float16,float16,0,0.1509999990463257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,1,128,1,float16,fp8,0,0.14684000015258789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,1,128,1,fp8,fp8,0,0.14893120527267456
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,2,128,1,float16,fp8,0,0.1745952010154724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,2,128,1,float16,float16,0,0.17955520153045654
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,2,128,1,fp8,fp8,0,0.17434879541397094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,4,128,1,float16,float16,0,0.1351696014404297
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,4,128,1,float16,fp8,0,0.13264960050582886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,4,128,1,fp8,fp8,0,0.1320863962173462
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,1,128,1,float16,float16,0,0.08858879804611205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,1,128,1,float16,fp8,0,0.08269760012626648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,1,128,1,fp8,fp8,0,0.0838927984237671
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,2,128,1,float16,float16,0,0.10410399436950683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,2,128,1,float16,fp8,0,0.09880639910697937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,2,128,1,fp8,fp8,0,0.09949280023574829
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,4,128,1,float16,float16,0,0.07925760149955749
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,4,128,1,float16,fp8,0,0.07383040189743043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,4,128,1,fp8,fp8,0,0.07275519967079162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,1,128,1,float16,float16,0,0.05407840013504028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,1,128,1,float16,fp8,0,0.0545199990272522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,1,128,1,fp8,fp8,0,0.05489919781684875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,2,128,1,float16,float16,0,0.06043040156364441
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,2,128,1,float16,fp8,0,0.06098560094833374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,2,128,1,fp8,fp8,0,0.0610975980758667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,4,128,1,float16,float16,0,0.06282079815864564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,4,128,1,float16,fp8,0,0.05959360003471374
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,4,128,1,fp8,fp8,0,0.06223359704017639
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,1,128,1,float16,float16,0,0.047916799783706665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,1,128,1,float16,fp8,0,0.049395200610160825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,1,128,1,fp8,fp8,0,0.04908480048179627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,2,128,1,float16,float16,0,0.05214560031890869
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,2,128,1,float16,fp8,0,0.053180801868438723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,2,128,1,fp8,fp8,0,0.054473602771759035
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,4,128,1,float16,float16,0,0.03996480107307434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,4,128,1,float16,fp8,0,0.04037440121173859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,4,128,1,fp8,fp8,0,0.040084800124168395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,1,128,1,float16,float16,0,0.03561440110206604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,1,128,1,float16,fp8,0,0.037415999174118045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,1,128,1,fp8,fp8,0,0.03709439933300018
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,2,128,1,float16,float16,0,0.03606399893760681
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,2,128,1,float16,fp8,0,0.03814879953861237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,2,128,1,fp8,fp8,0,0.0375328004360199
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,4,1,128,1,float16,float16,0,0.5605823993682861
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,4,1,128,1,float16,fp8,0,0.5427455902099609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,4,1,128,1,fp8,fp8,0,0.54475998878479
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,4,2,128,1,float16,float16,0,0.7166927814483642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,4,2,128,1,float16,fp8,0,0.6978591918945313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,1,128,1,float16,float16,0,0.2930448055267334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,4,128,1,float16,float16,0,0.5325439929962158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,4,2,128,1,fp8,fp8,0,0.6974880218505859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,4,128,1,float16,fp8,0,0.5164544105529785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,4,128,1,fp8,fp8,0,0.5212448120117188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,1,128,1,float16,fp8,0,0.2906255960464478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,1,128,1,fp8,fp8,0,0.29022719860076907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,2,128,1,float16,float16,0,0.37325599193573
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,2,128,1,float16,fp8,0,0.3622735977172852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,2,128,1,fp8,fp8,0,0.3652415990829468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,4,128,1,float16,float16,0,0.2805808067321777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,4,128,1,float16,fp8,0,0.27313759326934817
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,4,128,1,fp8,fp8,0,0.27532639503479006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,1,128,1,float16,float16,0,0.16341439485549927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,1,128,1,float16,fp8,0,0.16180959939956666
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,1,128,1,fp8,fp8,0,0.16159839630126954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,2,128,1,float16,float16,0,0.20062079429626464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,2,128,1,float16,fp8,0,0.19800159931182862
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,2,128,1,fp8,fp8,0,0.1982815980911255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,4,128,1,float16,float16,0,0.15408639907836913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,4,128,1,float16,fp8,0,0.15407999753952026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,4,128,1,fp8,fp8,0,0.15328160524368287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,1,128,1,float16,float16,0,0.09502559900283813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,1,128,1,float16,fp8,0,0.09341279864311218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,1,128,1,fp8,fp8,0,0.09036960005760193
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,4,128,1,fp8,fp8,0,0.08312000036239624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,2,128,1,float16,float16,0,0.11550559997558593
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,2,128,1,float16,fp8,0,0.11435199975967407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,2,128,1,fp8,fp8,0,0.11452480554580688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,4,128,1,float16,float16,0,0.09308159947395325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,4,128,1,float16,fp8,0,0.08471999764442444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,1,128,1,float16,float16,0,0.057222402095794676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,1,128,1,float16,fp8,0,0.05707520246505737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,1,128,1,fp8,fp8,0,0.057283198833465575
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,2,128,1,float16,float16,0,0.06923040151596069
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,2,128,1,float16,fp8,0,0.06469119787216186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,2,128,1,fp8,fp8,0,0.06466079950332641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,4,128,1,float16,float16,0,0.05089920163154602
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,4,128,1,float16,fp8,0,0.051288002729415895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,4,128,1,fp8,fp8,0,0.05132319927215576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,1,128,1,float16,float16,0,0.03850080072879791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,1,128,1,float16,fp8,0,0.03992800116539001
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,1,128,1,fp8,fp8,0,0.03984639942646027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,2,128,1,float16,float16,0,0.04280160069465637
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,2,128,1,float16,fp8,0,0.04416959881782532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,2,128,1,fp8,fp8,0,0.04421280026435852
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,4,128,1,float16,float16,0,0.04153760075569153
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,4,128,1,float16,fp8,0,0.04276959896087647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,4,128,1,fp8,fp8,0,0.043377599120140074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,1,128,1,float16,float16,0,0.036652800440788266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,1,128,1,float16,fp8,0,0.03796800076961517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,1,128,1,fp8,fp8,0,0.03834879994392395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,2,128,1,float16,float16,0,0.03720319867134094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,2,128,1,float16,fp8,0,0.038755199313163756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,2,128,1,fp8,fp8,0,0.03856799900531769
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,4,128,1,float16,float16,0,0.032476800680160525
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,4,128,1,float16,fp8,0,0.03311200141906738
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,4,128,1,fp8,fp8,0,0.033030399680137636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,1,128,1,float16,float16,0,0.031809601187705996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,1,128,1,float16,fp8,0,0.031860798597335815
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,1,128,1,fp8,fp8,0,0.03184320032596588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,2,128,1,float16,float16,0,0.03204799890518188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,2,128,1,float16,fp8,0,0.03211359977722168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,2,128,1,fp8,fp8,0,0.03219360113143921
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,4,1,128,1,float16,float16,0,0.405836820602417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,4,1,128,1,float16,fp8,0,0.40565919876098633
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,4,1,128,1,fp8,fp8,0,0.4045663833618164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,4,2,128,1,float16,float16,0,0.5630832195281983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,4,2,128,1,float16,fp8,0,0.5565775871276856
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,4,2,128,1,fp8,fp8,0,0.5570703983306885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,4,128,1,float16,float16,0,0.4538447856903076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,4,128,1,float16,fp8,0,0.44489760398864747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,4,128,1,fp8,fp8,0,0.4468095779418945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,1,128,1,float16,float16,0,0.21697280406951905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,1,128,1,float16,fp8,0,0.21481759548187257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,1,128,1,fp8,fp8,0,0.21528000831604005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,2,128,1,float16,float16,0,0.2950383901596069
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,2,128,1,float16,fp8,0,0.2915247917175293
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,2,128,1,fp8,fp8,0,0.2913536071777344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,4,128,1,float16,fp8,0,0.23421759605407716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,4,128,1,float16,float16,0,0.2407072067260742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,4,128,1,fp8,fp8,0,0.23499040603637694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,1,128,1,float16,float16,0,0.12114720344543457
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,4,128,1,float16,float16,0,0.1336143970489502
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,4,128,1,float16,fp8,0,0.13096799850463867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,1,128,1,float16,fp8,0,0.12179679870605468
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,1,128,1,fp8,fp8,0,0.12189760208129882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,2,128,1,float16,float16,0,0.1602687954902649
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,2,128,1,float16,fp8,0,0.15914560556411744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,2,128,1,fp8,fp8,0,0.15934239625930785
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,4,128,1,fp8,fp8,0,0.13046720027923583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,1,128,1,float16,float16,0,0.07368959784507752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,1,128,1,float16,fp8,0,0.06963520050048828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,1,128,1,fp8,fp8,0,0.06958079934120179
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,2,128,1,float16,float16,0,0.09313600063323975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,2,128,1,float16,fp8,0,0.09246879816055298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,2,128,1,fp8,fp8,0,0.09247360229492188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,4,128,1,float16,float16,0,0.0778880000114441
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,4,128,1,float16,fp8,0,0.0700432002544403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,4,128,1,fp8,fp8,0,0.07166720032691956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,1,128,1,float16,float16,0,0.043966400623321536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,1,128,1,float16,fp8,0,0.04387199878692627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,1,128,1,fp8,fp8,0,0.0438511997461319
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,2,128,1,float16,float16,0,0.053091198205947876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,2,128,1,float16,fp8,0,0.05135840177536011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,2,128,1,fp8,fp8,0,0.05151680111885071
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,4,128,1,float16,float16,0,0.04158399999141693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,4,128,1,float16,fp8,0,0.04281120002269745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,4,128,1,fp8,fp8,0,0.04281280040740967
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,1,128,1,float16,float16,0,0.02942880094051361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,1,128,1,float16,fp8,0,0.031036800146102904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,1,128,1,fp8,fp8,0,0.031004801392555237
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,2,128,1,float16,float16,0,0.034004798531532286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,2,128,1,float16,fp8,0,0.03535839915275574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,2,128,1,fp8,fp8,0,0.0352400004863739
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,4,128,1,float16,float16,0,0.033723199367523195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,4,128,1,float16,fp8,0,0.0350383996963501
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,4,128,1,fp8,fp8,0,0.034811198711395264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,1,128,1,float16,float16,0,0.028295999765396117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,1,128,1,float16,fp8,0,0.03023360073566437
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,1,128,1,fp8,fp8,0,0.029732799530029295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,2,128,1,float16,float16,0,0.029120001196861266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,2,128,1,float16,fp8,0,0.030564799904823303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,2,128,1,fp8,fp8,0,0.03112800121307373
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,4,128,1,float16,float16,0,0.029116800427436827
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,4,128,1,float16,fp8,0,0.03049280047416687
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,4,128,1,fp8,fp8,0,0.03105599880218506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,1,128,1,float16,float16,0,0.02757920026779175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,1,128,1,float16,fp8,0,0.029366400837898255
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,1,128,1,fp8,fp8,0,0.029582399129867553
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,2,128,1,float16,float16,0,0.027902400493621825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,2,128,1,float16,fp8,0,0.02990399897098541
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,2,128,1,fp8,fp8,0,0.029868799448013305
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,4,128,1,float16,float16,0,0.027475199103355406
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,4,128,1,float16,fp8,0,0.02956480085849762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,4,128,1,fp8,fp8,0,0.030380800366401672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,1,128,1,float16,float16,0,0.027470400929450987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,1,128,1,float16,fp8,0,0.028963199257850646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,1,128,1,fp8,fp8,0,0.02900800108909607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,2,128,1,float16,float16,0,0.027209600806236266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,2,128,1,float16,fp8,0,0.02906239926815033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,2,128,1,fp8,fp8,0,0.029126399755477907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,4,1,128,1,float16,float16,0,0.19458240270614624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,4,1,128,1,float16,fp8,0,0.1941264033317566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,4,1,128,1,fp8,fp8,0,0.19431519508361816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,4,2,128,1,float16,float16,0,0.27485439777374265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,4,2,128,1,float16,fp8,0,0.269817590713501
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,4,2,128,1,fp8,fp8,0,0.26938400268554685
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,4,128,1,float16,float16,0,0.23265280723571777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,4,128,1,float16,fp8,0,0.22039999961853027
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,4,128,1,fp8,fp8,0,0.2202608108520508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,1,128,1,float16,float16,0,0.11050399541854858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,1,128,1,float16,fp8,0,0.11048320531845093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,1,128,1,fp8,fp8,0,0.11105920076370239
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,2,128,1,float16,float16,0,0.15023200511932372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,2,128,1,float16,fp8,0,0.14762239456176757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,2,128,1,fp8,fp8,0,0.14770079851150514
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,4,128,1,float16,float16,0,0.1286352038383484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,4,128,1,float16,fp8,0,0.12177759408950806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,4,128,1,fp8,fp8,0,0.12182559967041015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,1,128,1,float16,float16,0,0.06735680103302003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,1,128,1,float16,fp8,0,0.06241440176963806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,1,128,1,fp8,fp8,0,0.06409599781036376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,2,128,1,float16,float16,0,0.08731039762496948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,2,128,1,float16,fp8,0,0.0854319989681244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,2,128,1,fp8,fp8,0,0.08531839847564697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,4,128,1,float16,float16,0,0.07343360185623168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,4,128,1,float16,fp8,0,0.06445599794387817
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,4,128,1,fp8,fp8,0,0.06356160044670105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,1,128,1,float16,float16,0,0.038094401359558105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,1,128,1,float16,fp8,0,0.0391184002161026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,1,128,1,fp8,fp8,0,0.03905600011348724
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,2,128,1,float16,float16,0,0.048332801461219786
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,2,128,1,float16,fp8,0,0.04635039865970612
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,2,128,1,fp8,fp8,0,0.04671359956264496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,4,128,1,float16,float16,0,0.03880000114440918
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,4,128,1,float16,fp8,0,0.038980799913406375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,4,128,1,fp8,fp8,0,0.03900800049304962
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,1,128,1,float16,float16,0,0.026630398631095887
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,1,128,1,float16,fp8,0,0.027342399954795836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,1,128,1,fp8,fp8,0,0.02723039984703064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,2,128,1,float16,float16,0,0.03078559935092926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,2,128,1,float16,fp8,0,0.03176479935646057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,2,128,1,fp8,fp8,0,0.03159680068492889
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,4,128,1,float16,float16,0,0.03056960105895996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,4,128,1,float16,fp8,0,0.03129439949989319
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,4,128,1,fp8,fp8,0,0.031563198566436766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,1,128,1,float16,float16,0,0.02534399926662445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,1,128,1,float16,fp8,0,0.026241600513458252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,1,128,1,fp8,fp8,0,0.02601119875907898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,2,128,1,float16,float16,0,0.026283198595047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,2,128,1,float16,fp8,0,0.026966398954391478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,2,128,1,fp8,fp8,0,0.027024000883102417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,4,128,1,float16,float16,0,0.02571519911289215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,4,128,1,float16,fp8,0,0.02680320143699646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,4,128,1,fp8,fp8,0,0.026719999313354493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,1,128,1,float16,float16,0,0.024486400187015533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,1,128,1,float16,fp8,0,0.02550559937953949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,1,128,1,fp8,fp8,0,0.025811201333999632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,2,128,1,float16,float16,0,0.025038400292396547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,2,128,1,float16,fp8,0,0.026241600513458252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,2,128,1,fp8,fp8,0,0.025729599595069885
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,4,128,1,float16,float16,0,0.024582399427890776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,4,128,1,float16,fp8,0,0.025953599810600282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,2,128,1,fp8,fp8,0,0.025566399097442627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,4,128,1,fp8,fp8,0,0.025865599513053894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,1,128,1,float16,float16,0,0.02450399994850159
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,1,128,1,float16,fp8,0,0.025545600056648254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,1,128,1,fp8,fp8,0,0.025374400615692138
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,2,128,1,float16,float16,0,0.024318400025367736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,2,128,1,float16,fp8,0,0.025319999456405638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,4,128,1,float16,float16,0,0.02330400049686432
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,4,128,1,float16,fp8,0,0.024188800156116484
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,4,128,1,fp8,fp8,0,0.024412800371646882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,1,128,1,float16,float16,0,0.02306399941444397
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,1,128,1,float16,fp8,0,0.024051199853420257
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,1,128,1,fp8,fp8,0,0.024030399322509766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,2,128,1,float16,float16,0,0.023151999711990355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,2,128,1,float16,fp8,0,0.024193599820137024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,2,128,1,fp8,fp8,0,0.024297599494457246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,4,1,128,1,float16,float16,0,0.10920480489730836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,4,1,128,1,float16,fp8,0,0.10932799577713012
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,4,1,128,1,fp8,fp8,0,0.10906239748001098
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,4,2,128,1,float16,float16,0,0.1507439970970154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,4,2,128,1,float16,fp8,0,0.1461392045021057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,4,2,128,1,fp8,fp8,0,0.14600640535354614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,4,128,1,float16,float16,0,0.12851840257644653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,4,128,1,float16,fp8,0,0.12003040313720703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,4,128,1,fp8,fp8,0,0.11902079582214356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,1,128,1,float16,float16,0,0.06460480093955993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,1,128,1,float16,fp8,0,0.060247999429702756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,1,128,1,fp8,fp8,0,0.05979679822921753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,2,128,1,float16,float16,0,0.08577119708061218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,2,128,1,float16,fp8,0,0.0818656027317047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,2,128,1,fp8,fp8,0,0.08143039941787719
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,4,128,1,float16,float16,0,0.07181439995765686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,4,128,1,float16,fp8,0,0.06000000238418579
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,4,128,1,fp8,fp8,0,0.06113280057907104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,1,128,1,float16,float16,0,0.035571199655532834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,1,128,1,float16,fp8,0,0.03627200126647949
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,1,128,1,fp8,fp8,0,0.03642399907112122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,4,128,1,float16,fp8,0,0.037415999174118045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,2,128,1,float16,float16,0,0.04683519899845123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,2,128,1,float16,fp8,0,0.04418559968471527
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,2,128,1,fp8,fp8,0,0.04333280026912689
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,4,128,1,float16,float16,0,0.03815360069274902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,4,128,1,fp8,fp8,0,0.03761439919471741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,2,128,1,fp8,fp8,0,0.03038559854030609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,1,128,1,float16,float16,0,0.025884801149368288
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,4,128,1,float16,float16,0,0.028883200883865357
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,4,128,1,float16,fp8,0,0.029827201366424562
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,1,128,1,float16,fp8,0,0.02613599896430969
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,1,128,1,fp8,fp8,0,0.02619200050830841
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,2,128,1,float16,float16,0,0.030161601305007935
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,2,128,1,float16,fp8,0,0.03047040104866028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,4,128,1,fp8,fp8,0,0.029495999217033386
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,1,128,1,float16,float16,0,0.0236272007226944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,1,128,1,float16,fp8,0,0.024846400320529937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,1,128,1,fp8,fp8,0,0.024635200202465058
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,2,128,1,float16,float16,0,0.02481919974088669
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,2,128,1,float16,fp8,0,0.025835201144218445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,2,128,1,fp8,fp8,0,0.025523200631141663
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,4,128,1,float16,float16,0,0.024875199794769286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,4,128,1,float16,fp8,0,0.024903999269008638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,4,128,1,fp8,fp8,0,0.02556000053882599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,1,128,1,float16,float16,0,0.023292799293994904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,1,128,1,float16,fp8,0,0.023871999979019166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,1,128,1,fp8,fp8,0,0.024424000084400176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,2,128,1,float16,float16,0,0.023822399973869323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,2,128,1,float16,fp8,0,0.024240000545978545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,2,128,1,fp8,fp8,0,0.023950399458408357
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,4,128,1,float16,float16,0,0.023206399381160737
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,4,128,1,float16,fp8,0,0.023950399458408357
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,4,128,1,fp8,fp8,0,0.0237744003534317
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,1,128,1,float16,float16,0,0.023001599311828613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,1,128,1,float16,fp8,0,0.0235615998506546
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,1,128,1,fp8,fp8,0,0.02356480062007904
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,2,128,1,float16,float16,0,0.02269279956817627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,2,128,1,float16,fp8,0,0.023489600419998168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,2,128,1,fp8,fp8,0,0.02351839989423752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,4,128,1,float16,float16,0,0.021559999883174898
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,4,128,1,float16,fp8,0,0.022329600155353548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,4,128,1,fp8,fp8,0,0.022390399873256684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,1,128,1,float16,float16,0,0.021478399634361267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,1,128,1,float16,fp8,0,0.022232000529766083
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,1,128,1,fp8,fp8,0,0.022033600509166716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,2,128,1,float16,float16,0,0.021566399931907655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,2,128,1,float16,fp8,0,0.022166399657726286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,2,128,1,fp8,fp8,0,0.022145600616931917
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,4,128,1,float16,float16,0,0.021104000508785248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,4,128,1,float16,fp8,0,0.021668800711631776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,4,128,1,fp8,fp8,0,0.021542400121688843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,1,128,1,float16,float16,0,0.020735999941825865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,1,128,1,float16,fp8,0,0.021278400719165803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,1,128,1,fp8,fp8,0,0.021768000721931458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,2,128,1,float16,float16,0,0.020803199708461763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,2,128,1,float16,fp8,0,0.021491199731826782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,2,128,1,fp8,fp8,0,0.02163040041923523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,4,1,128,1,float16,float16,0,0.06622400283813476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,4,1,128,1,float16,fp8,0,0.061699199676513675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,4,1,128,1,fp8,fp8,0,0.062027198076248166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,4,2,128,1,float16,float16,0,0.08630719780921936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,4,2,128,1,float16,fp8,0,0.08350399732589722
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,4,2,128,1,fp8,fp8,0,0.08353279829025269
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,4,128,1,float16,float16,0,0.07957919836044311
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,4,128,1,float16,fp8,0,0.07047039866447449
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,4,128,1,fp8,fp8,0,0.07092800140380859
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,1,128,1,float16,float16,0,0.03694559931755066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,1,128,1,float16,fp8,0,0.03696480095386505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,1,128,1,fp8,fp8,0,0.03739199936389923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,2,128,1,float16,float16,0,0.04782080054283142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,2,128,1,float16,fp8,0,0.04418720006942749
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,2,128,1,fp8,fp8,0,0.044566398859024046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,4,128,1,float16,float16,0,0.0418287992477417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,4,128,1,float16,fp8,0,0.04222080111503601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,4,128,1,fp8,fp8,0,0.042310398817062375
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,1,128,1,float16,float16,0,0.025972801446914672
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,1,128,1,float16,fp8,0,0.026881599426269533
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,1,128,1,fp8,fp8,0,0.026683199405670165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,2,128,1,float16,float16,0,0.030456000566482545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,2,128,1,float16,fp8,0,0.031091201305389404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,2,128,1,fp8,fp8,0,0.031145599484443665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,4,128,1,float16,float16,0,0.029100799560546876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,4,128,1,float16,fp8,0,0.029628801345825195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,4,128,1,fp8,fp8,0,0.02980000078678131
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,1,128,1,float16,float16,0,0.02391359955072403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,1,128,1,float16,fp8,0,0.025009599328041077
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,1,128,1,fp8,fp8,0,0.024963200092315674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,2,128,1,float16,float16,0,0.02481279969215393
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,2,128,1,float16,fp8,0,0.0259552001953125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,2,128,1,fp8,fp8,0,0.025911998748779298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,4,128,1,float16,float16,0,0.023992000520229338
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,4,128,1,float16,fp8,0,0.02487040013074875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,4,128,1,fp8,fp8,0,0.024721600115299225
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,1,128,1,float16,float16,0,0.02344159930944443
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,1,128,1,float16,fp8,0,0.024054400622844696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,1,128,1,fp8,fp8,0,0.023844799399375914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,2,128,1,float16,float16,0,0.02366719990968704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,2,128,1,float16,fp8,0,0.024542400240898134
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,2,128,1,fp8,fp8,0,0.02433920055627823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,4,128,1,float16,float16,0,0.022888000309467315
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,4,128,1,float16,fp8,0,0.024195200204849242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,4,128,1,fp8,fp8,0,0.02343679964542389
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,1,128,1,float16,float16,0,0.02292799949645996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,1,128,1,float16,fp8,0,0.023731200397014617
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,1,128,1,fp8,fp8,0,0.02359039932489395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,2,128,1,float16,float16,0,0.022859199345111846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,2,128,1,float16,fp8,0,0.02350880056619644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,2,128,1,fp8,fp8,0,0.02380319982767105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,4,128,1,float16,float16,0,0.021167999505996703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,4,128,1,float16,fp8,0,0.022358399629592896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,4,128,1,fp8,fp8,0,0.0221343994140625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,1,128,1,float16,float16,0,0.021561600267887115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,1,128,1,float16,fp8,0,0.022015999257564544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,1,128,1,fp8,fp8,0,0.022033600509166716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,2,128,1,float16,float16,0,0.021667200326919555
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,2,128,1,float16,fp8,0,0.022035199403762817
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,2,128,1,fp8,fp8,0,0.022078399360179902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,4,128,1,float16,float16,0,0.020721599459648132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,4,128,1,float16,fp8,0,0.021430400013923646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,4,128,1,fp8,fp8,0,0.021355199813842773
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,1,128,1,float16,float16,0,0.021009600162506102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,1,128,1,float16,fp8,0,0.021799999475479125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,1,128,1,fp8,fp8,0,0.021358400583267212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,2,128,1,float16,float16,0,0.020572799444198608
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,2,128,1,float16,fp8,0,0.021750399470329286
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,2,128,1,fp8,fp8,0,0.02166240066289902
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,4,128,1,float16,float16,0,0.02030719965696335
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,4,128,1,float16,fp8,0,0.021078400313854218
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,4,128,1,fp8,fp8,0,0.021374399960041045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,1,128,1,float16,float16,0,0.02045599967241287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,1,128,1,float16,fp8,0,0.021110400557518005
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,1,128,1,fp8,fp8,0,0.02143840044736862
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,2,128,1,float16,float16,0,0.020526400208473204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,2,128,1,float16,fp8,0,0.02163040041923523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,2,128,1,fp8,fp8,0,0.021460799872875212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,4,1,128,1,float16,float16,0,0.03896960020065308
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,4,1,128,1,float16,fp8,0,0.039241600036621097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,4,1,128,1,fp8,fp8,0,0.039875200390815733
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,4,2,128,1,float16,float16,0,0.05619840025901794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,4,2,128,1,float16,fp8,0,0.05526559948921204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,4,2,128,1,fp8,fp8,0,0.0553600013256073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,4,128,1,float16,float16,0,0.04999040067195892
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,4,128,1,float16,fp8,0,0.05246080160140991
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,4,128,1,fp8,fp8,0,0.0523792028427124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,1,128,1,float16,float16,0,0.027089598774909972
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,1,128,1,float16,fp8,0,0.028230398893356323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,1,128,1,fp8,fp8,0,0.028030401468276976
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,2,128,1,float16,float16,0,0.0350928008556366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,2,128,1,float16,fp8,0,0.036504000425338745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,1,128,1,fp8,fp8,0,0.025879999995231627
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,2,128,1,fp8,fp8,0,0.03637279868125916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,4,128,1,float16,float16,0,0.03321279883384705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,4,128,1,float16,fp8,0,0.03513439893722534
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,4,128,1,fp8,fp8,0,0.034897598624229434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,1,128,1,float16,float16,0,0.024508799612522125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,1,128,1,float16,fp8,0,0.025992000102996828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,2,128,1,float16,float16,0,0.025726398825645445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,2,128,1,float16,fp8,0,0.02656799852848053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,2,128,1,fp8,fp8,0,0.026417601108551025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,4,128,1,float16,float16,0,0.024371199309825897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,4,128,1,float16,fp8,0,0.025515198707580566
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,4,128,1,fp8,fp8,0,0.02550399899482727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,1,128,1,float16,float16,0,0.02358720004558563
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,1,128,1,float16,fp8,0,0.024732799828052522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,1,128,1,fp8,fp8,0,0.024635200202465058
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,2,128,1,float16,float16,0,0.02401760071516037
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,2,128,1,float16,fp8,0,0.02454719990491867
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,2,128,1,fp8,fp8,0,0.02414399981498718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,4,128,1,float16,float16,0,0.023155200481414794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,4,128,1,float16,fp8,0,0.023940800130367278
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,4,128,1,fp8,fp8,0,0.023609599471092223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,1,128,1,float16,float16,0,0.023161600530147552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,1,128,1,float16,fp8,0,0.023665599524974823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,1,128,1,fp8,fp8,0,0.02402399927377701
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,2,128,1,float16,float16,0,0.0226623997092247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,2,128,1,float16,fp8,0,0.023313599824905395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,2,128,1,fp8,fp8,0,0.023878400027751923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,4,128,1,float16,float16,0,0.021712000668048858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,4,128,1,float16,fp8,0,0.022329600155353548
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,4,128,1,fp8,fp8,0,0.02276480048894882
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,1,128,1,float16,float16,0,0.021686400473117828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,1,128,1,float16,fp8,0,0.022337600588798523
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,1,128,1,fp8,fp8,0,0.02236640006303787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,2,128,1,float16,float16,0,0.02139520049095154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,2,128,1,float16,fp8,0,0.022334399819374084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,2,128,1,fp8,fp8,0,0.022291199862957002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,4,128,1,float16,float16,0,0.020555199682712556
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,4,128,1,float16,fp8,0,0.021740800142288207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,4,128,1,fp8,fp8,0,0.02165919989347458
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,1,128,1,float16,float16,0,0.02075359970331192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,1,128,1,float16,fp8,0,0.021644799411296843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,1,128,1,fp8,fp8,0,0.02139520049095154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,2,128,1,float16,float16,0,0.020683200657367708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,2,128,1,float16,fp8,0,0.021211199462413788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,2,128,1,fp8,fp8,0,0.02122880071401596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,4,128,1,float16,float16,0,0.020388799905776977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,4,128,1,float16,fp8,0,0.021356800198554994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,4,128,1,fp8,fp8,0,0.02129279971122742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,1,128,1,float16,float16,0,0.020294399559497835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,1,128,1,float16,fp8,0,0.02136480063199997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,1,128,1,fp8,fp8,0,0.02147040069103241
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,2,128,1,float16,float16,0,0.020308800041675568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,2,128,1,float16,fp8,0,0.02128800004720688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,2,128,1,fp8,fp8,0,0.021236799657344818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,4,128,1,float16,float16,0,0.02040639966726303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,4,128,1,float16,fp8,0,0.021035200357437132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,4,128,1,fp8,fp8,0,0.021241599321365358
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,1,128,1,float16,float16,0,0.020304000377655028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,1,128,1,float16,fp8,0,0.02119999974966049
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,1,128,1,fp8,fp8,0,0.021451200544834136
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,2,128,1,float16,float16,0,0.020308800041675568
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,2,128,1,float16,fp8,0,0.02115200012922287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,2,128,1,fp8,fp8,0,0.021049599349498748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,4,1,128,1,float16,float16,0,0.03276639878749847
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,4,1,128,1,float16,fp8,0,0.03443520069122315
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,4,1,128,1,fp8,fp8,0,0.03482399880886078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,4,2,128,1,float16,float16,0,0.04440160095691681
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,4,2,128,1,float16,fp8,0,0.04771359860897064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,4,2,128,1,fp8,fp8,0,0.047705599665641786
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,4,128,1,float16,float16,0,0.04153439998626709
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,4,128,1,float16,fp8,0,0.04532000124454498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,4,128,1,fp8,fp8,0,0.04519839882850647
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,1,128,1,float16,float16,0,0.025513601303100587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,1,128,1,float16,fp8,0,0.026929599046707154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,1,128,1,fp8,fp8,0,0.02707839906215668
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,2,128,1,float16,float16,0,0.03062720000743866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,2,128,1,float16,fp8,0,0.03156639933586121
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,2,128,1,fp8,fp8,0,0.03198400139808655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,4,128,1,float16,float16,0,0.029123198986053467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,4,128,1,float16,fp8,0,0.030511999130249025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,4,128,1,fp8,fp8,0,0.030748799443244934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,1,128,1,float16,float16,0,0.02449920028448105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,1,128,1,float16,fp8,0,0.025260800123214723
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,1,128,1,fp8,fp8,0,0.024771200120449068
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,2,128,1,float16,float16,0,0.024596799910068513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,2,128,1,float16,fp8,0,0.025329598784446718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,2,128,1,fp8,fp8,0,0.025435200333595274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,4,128,1,float16,float16,0,0.02364159971475601
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,4,128,1,float16,fp8,0,0.02473440021276474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,4,128,1,fp8,fp8,0,0.02476159930229187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,1,128,1,float16,float16,0,0.02362080067396164
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,1,128,1,float16,fp8,0,0.023865599930286408
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,1,128,1,fp8,fp8,0,0.024302400648593903
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,2,128,1,float16,float16,0,0.023499199748039247
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,2,128,1,float16,fp8,0,0.024182400107383727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,2,128,1,fp8,fp8,0,0.023902399837970732
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,4,128,1,float16,float16,0,0.02191520035266876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,4,128,1,float16,fp8,0,0.022924800217151643
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,4,128,1,fp8,fp8,0,0.022894400358200073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,1,128,1,float16,float16,0,0.02129279971122742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,1,128,1,float16,fp8,0,0.022291199862957002
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,1,128,1,fp8,fp8,0,0.02227199971675873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,2,128,1,float16,float16,0,0.021748800575733186
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,2,128,1,float16,fp8,0,0.022622400522232057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,2,128,1,fp8,fp8,0,0.02266719937324524
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,4,128,1,float16,float16,0,0.021116800606250763
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,4,128,1,float16,fp8,0,0.021985599398612977
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,4,128,1,fp8,fp8,0,0.02200320065021515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,1,128,1,float16,float16,0,0.0207056000828743
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,1,128,1,float16,fp8,0,0.021488000452518464
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,1,128,1,fp8,fp8,0,0.021220800280570985
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,2,128,1,float16,float16,0,0.020712000131607056
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,2,128,1,float16,fp8,0,0.02152319997549057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,2,128,1,fp8,fp8,0,0.021641600131988525
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,4,128,1,float16,float16,0,0.020531199872493744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,4,128,1,float16,fp8,0,0.02106879949569702
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,4,128,1,fp8,fp8,0,0.02159679979085922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,1,128,1,float16,float16,0,0.020318399369716644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,1,128,1,float16,fp8,0,0.021033599972724915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,1,128,1,fp8,fp8,0,0.0210207998752594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,2,128,1,float16,float16,0,0.020295999944210052
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,2,128,1,float16,fp8,0,0.02123039960861206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,2,128,1,fp8,fp8,0,0.02117599993944168
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,4,128,1,float16,float16,0,0.01995840072631836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,4,128,1,float16,fp8,0,0.021380800008773803
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,4,128,1,fp8,fp8,0,0.021155199408531188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,1,128,1,float16,float16,0,0.019969600439071655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,1,128,1,float16,fp8,0,0.020916800200939178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,1,128,1,fp8,fp8,0,0.021164800226688384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,2,128,1,float16,float16,0,0.020403200387954713
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,1,128,1,float16,fp8,0,0.02106720060110092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,2,128,1,float16,fp8,0,0.020980800688266753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,1,128,1,fp8,fp8,0,0.021012799441814424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,2,128,1,float16,float16,0,0.020158399641513825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,2,128,1,fp8,fp8,0,0.020983999967575072
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,4,128,1,float16,float16,0,0.020294399559497835
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,4,128,1,float16,fp8,0,0.021111999452114106
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,4,128,1,fp8,fp8,0,0.020883199572563172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,1,128,1,float16,float16,0,0.020017600059509276
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,2,128,1,float16,fp8,0,0.02120800018310547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,2,128,1,fp8,fp8,0,0.02098879963159561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,2,1,128,1,float16,float16,0,1.9061519622802734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,2,2,128,1,float16,float16,0,1.017591953277588
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,2,1,128,1,float16,fp8,0,1.72391357421875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,2,2,128,1,float16,fp8,0,0.9606975555419922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,2,2,128,1,fp8,fp8,0,0.9658432006835938
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,2,1,128,1,fp8,fp8,0,1.734889602661133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,2,1,128,1,float16,float16,0,0.939361572265625
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,2,2,128,1,float16,fp8,0,0.4974544048309326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,2,2,128,1,float16,float16,0,0.5158703804016114
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,2,1,128,1,float16,fp8,0,0.9493951797485352
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,2,2,128,1,fp8,fp8,0,0.5012239933013916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,2,1,128,1,fp8,fp8,0,0.9178832054138184
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,2,2,128,1,float16,fp8,0,0.2515727996826172
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,2,1,128,1,float16,float16,0,0.47035999298095704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,2,1,128,1,float16,fp8,0,0.46603999137878416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,2,1,128,1,fp8,fp8,0,0.45599517822265623
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,2,2,128,1,float16,float16,0,0.2653136014938354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,2,2,128,1,fp8,fp8,0,0.2538784027099609
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,2,1,128,1,float16,float16,0,0.2453632116317749
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,2,1,128,1,float16,fp8,0,0.23312959671020508
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,2,1,128,1,fp8,fp8,0,0.2346719980239868
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,2,1,128,1,float16,float16,0,1.108283233642578
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,2,2,128,1,float16,float16,0,0.668617582321167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,2,1,128,1,float16,fp8,0,1.0332223892211914
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,2,2,128,1,float16,fp8,0,0.5986144065856933
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,2,1,128,1,fp8,fp8,0,1.0340928077697753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,2,2,128,1,fp8,fp8,0,0.6086736202239991
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,2,1,128,1,float16,float16,0,0.5672287940979004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,2,1,128,1,float16,fp8,0,0.5434368133544922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,2,1,128,1,fp8,fp8,0,0.5479423999786377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,2,2,128,1,float16,float16,0,0.3576080083847046
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,2,2,128,1,float16,fp8,0,0.32026400566101076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,2,1,128,1,fp8,fp8,0,0.2866624116897583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,2,2,128,1,fp8,fp8,0,0.31988160610198973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,2,1,128,1,float16,float16,0,0.3102848052978516
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,2,1,128,1,float16,fp8,0,0.2940880060195923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,2,2,128,1,float16,float16,0,0.2065279960632324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,2,2,128,1,float16,fp8,0,0.190556800365448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,2,2,128,1,fp8,fp8,0,0.1911743998527527
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,2,1,128,1,float16,float16,0,0.1838479995727539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,2,1,128,1,float16,fp8,0,0.17813760042190552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,2,1,128,1,fp8,fp8,0,0.18340640068054198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,2,1,128,1,float16,float16,0,0.8005760192871094
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,2,1,128,1,float16,fp8,0,0.7659887790679931
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,2,2,128,1,float16,float16,0,0.49597921371459963
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,2,1,128,1,fp8,fp8,0,0.7671311855316162
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,2,2,128,1,float16,fp8,0,0.4513535976409912
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,2,2,128,1,fp8,fp8,0,0.4451727867126465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,2,1,128,1,float16,float16,0,0.41403517723083494
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,2,1,128,1,float16,fp8,0,0.41069598197937013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,2,2,128,1,float16,float16,0,0.2925328016281128
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,2,1,128,1,fp8,fp8,0,0.4084767818450928
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,2,2,128,1,float16,fp8,0,0.23731839656829834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,2,2,128,1,fp8,fp8,0,0.23940320014953614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,2,1,128,1,float16,float16,0,0.22547199726104736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,2,1,128,1,float16,fp8,0,0.20886240005493165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,2,1,128,1,fp8,fp8,0,0.2079103946685791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,2,1,128,1,fp8,fp8,0,0.15128480195999144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,2,2,128,1,float16,float16,0,0.17313120365142823
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,2,2,128,1,float16,fp8,0,0.162390398979187
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,2,2,128,1,fp8,fp8,0,0.16565760374069213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,2,1,128,1,float16,float16,0,0.15615999698638916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,2,1,128,1,float16,fp8,0,0.15136159658432008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,2,1,128,1,float16,float16,0,1.0466943740844727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,2,2,128,1,float16,float16,0,0.6140575885772706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,2,1,128,1,float16,fp8,0,0.9909024238586426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,2,2,128,1,float16,fp8,0,0.5849247932434082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,2,1,128,1,fp8,fp8,0,0.9834624290466308
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,2,2,128,1,fp8,fp8,0,0.5831056118011475
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,2,1,128,1,float16,float16,0,0.5336175918579101
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,2,1,128,1,float16,fp8,0,0.5063199996948242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,2,2,128,1,float16,float16,0,0.3204927921295166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,2,1,128,1,fp8,fp8,0,0.5394224166870117
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,2,2,128,1,float16,fp8,0,0.3012480020523071
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,2,2,128,1,fp8,fp8,0,0.30399839878082274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,2,1,128,1,float16,float16,0,0.27494239807128906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,2,1,128,1,float16,fp8,0,0.26893439292907717
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,2,1,128,1,fp8,fp8,0,0.2705071926116943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,2,1,128,1,fp8,fp8,0,0.1362944006919861
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,2,2,128,1,float16,float16,0,0.16801919937133789
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,2,2,128,1,float16,fp8,0,0.15855200290679933
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,2,2,128,1,fp8,fp8,0,0.1570479989051819
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,2,1,128,1,float16,float16,0,0.1440351963043213
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,2,1,128,1,float16,fp8,0,0.13839679956436157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,2,2,128,1,float16,float16,0,0.1338479995727539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,2,2,128,1,float16,fp8,0,0.13384000062942505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,2,2,128,1,fp8,fp8,0,0.13441599607467652
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,2,1,128,1,float16,float16,0,0.12578879594802855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,2,1,128,1,float16,fp8,0,0.12744640111923217
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,2,1,128,1,fp8,fp8,0,0.12738399505615233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,2,2,128,1,float16,fp8,0,0.4100351810455322
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,2,1,128,1,float16,float16,0,0.6783567905426026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,2,1,128,1,float16,fp8,0,0.6512703895568848
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,2,1,128,1,fp8,fp8,0,0.6507343769073486
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,2,2,128,1,float16,float16,0,0.42816801071166993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,2,2,128,1,fp8,fp8,0,0.40359201431274416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,2,1,128,1,float16,float16,0,0.3680351972579956
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,2,1,128,1,float16,fp8,0,0.35741119384765624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,2,1,128,1,fp8,fp8,0,0.3549583911895752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,2,2,128,1,float16,float16,0,0.23319199085235595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,2,2,128,1,float16,fp8,0,0.22527039051055908
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,2,2,128,1,float16,fp8,0,0.1253872036933899
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,2,2,128,1,fp8,fp8,0,0.2242192029953003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,2,1,128,1,float16,float16,0,0.20619840621948243
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,2,1,128,1,float16,fp8,0,0.1946336030960083
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,2,1,128,1,fp8,fp8,0,0.19818400144577025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,2,2,128,1,float16,float16,0,0.13413439989089965
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,2,2,128,1,fp8,fp8,0,0.12341920137405396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,2,1,128,1,float16,float16,0,0.1137503981590271
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,2,1,128,1,float16,fp8,0,0.11034719944000244
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,2,1,128,1,fp8,fp8,0,0.11257439851760864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,2,1,128,1,float16,fp8,0,0.07485920190811157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,2,2,128,1,float16,float16,0,0.09118720293045043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,2,2,128,1,float16,fp8,0,0.08989920020103455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,2,2,128,1,fp8,fp8,0,0.09032480120658874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,2,1,128,1,float16,float16,0,0.07518399953842163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,2,1,128,1,float16,float16,0,0.6898303985595703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,2,1,128,1,fp8,fp8,0,0.07407680153846741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,2,1,128,1,float16,fp8,0,0.6626688003540039
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,2,1,128,1,fp8,fp8,0,0.6524159908294678
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,2,2,128,1,float16,float16,0,0.4504415988922119
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,2,2,128,1,float16,fp8,0,0.43004798889160156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,2,2,128,1,fp8,fp8,0,0.42761759757995604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,2,1,128,1,float16,float16,0,0.36850719451904296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,2,1,128,1,float16,fp8,0,0.35295040607452394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,2,1,128,1,fp8,fp8,0,0.3552623987197876
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,2,2,128,1,float16,float16,0,0.24544799327850342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,2,2,128,1,float16,fp8,0,0.23935039043426515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,2,2,128,1,fp8,fp8,0,0.23801760673522948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,2,1,128,1,float16,float16,0,0.20838239192962646
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,2,1,128,1,float16,fp8,0,0.20214879512786865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,2,1,128,1,fp8,fp8,0,0.20096800327301026
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,2,2,128,1,float16,float16,0,0.14725279808044434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,2,1,128,1,fp8,fp8,0,0.11730400323867798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,2,2,128,1,float16,fp8,0,0.13590400218963622
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,2,2,128,1,fp8,fp8,0,0.13648799657821656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,2,1,128,1,float16,float16,0,0.12446240186691285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,2,1,128,1,float16,fp8,0,0.11613600254058838
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,2,2,128,1,float16,float16,0,0.08923839926719665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,2,2,128,1,float16,fp8,0,0.0891215980052948
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,2,2,128,1,fp8,fp8,0,0.0893119990825653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,2,1,128,1,float16,float16,0,0.08075039982795715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,2,1,128,1,float16,fp8,0,0.08167679905891419
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,2,1,128,1,fp8,fp8,0,0.08122720122337342
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,2,2,128,1,float16,float16,0,0.06080160140991211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,2,2,128,1,float16,fp8,0,0.06068159937858582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,2,2,128,1,fp8,fp8,0,0.06125280261039734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,2,1,128,1,float16,float16,0,0.05671679973602295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,2,1,128,1,float16,fp8,0,0.0565168023109436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,2,1,128,1,fp8,fp8,0,0.05652160048484802
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,2,1,128,1,float16,float16,0,0.45142240524291993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,2,1,128,1,float16,fp8,0,0.43154082298278806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,2,2,128,1,float16,float16,0,0.30543200969696044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,2,1,128,1,fp8,fp8,0,0.4336719989776611
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,2,2,128,1,float16,fp8,0,0.29444160461425783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,2,2,128,1,fp8,fp8,0,0.29802401065826417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,2,1,128,1,float16,float16,0,0.2456144094467163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,2,1,128,1,float16,fp8,0,0.23921759128570558
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,2,1,128,1,fp8,fp8,0,0.24021759033203124
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,2,2,128,1,float16,float16,0,0.17435359954833984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,2,2,128,1,float16,fp8,0,0.16790720224380493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,2,2,128,1,fp8,fp8,0,0.1670624017715454
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,2,1,128,1,float16,float16,0,0.1428607940673828
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,2,1,128,1,float16,fp8,0,0.14032959938049316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,2,1,128,1,fp8,fp8,0,0.13479039669036866
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,2,2,128,1,float16,float16,0,0.1022271990776062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,2,2,128,1,float16,fp8,0,0.09490079879760742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,2,2,128,1,fp8,fp8,0,0.09429759979248047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,2,1,128,1,float16,float16,0,0.08198559880256653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,2,1,128,1,float16,fp8,0,0.08180480003356934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,2,1,128,1,fp8,fp8,0,0.08247680068016053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,2,2,128,1,float16,float16,0,0.07398560047149658
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,2,2,128,1,float16,fp8,0,0.07214400172233582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,2,2,128,1,fp8,fp8,0,0.07167680263519287
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,2,1,128,1,float16,float16,0,0.0629584014415741
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,2,1,128,1,float16,fp8,0,0.061972802877426146
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,2,1,128,1,fp8,fp8,0,0.06312479972839355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,2,2,128,1,float16,float16,0,0.04483039975166321
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,2,2,128,1,float16,fp8,0,0.045977601408958436
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,2,2,128,1,fp8,fp8,0,0.04594239890575409
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,2,1,128,1,float16,float16,0,0.042438399791717527
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,2,1,128,1,float16,fp8,0,0.04386720061302185
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,2,1,128,1,fp8,fp8,0,0.04365600049495697
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,2,1,128,1,float16,float16,0,0.4803455829620361
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,2,1,128,1,float16,fp8,0,0.460968017578125
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,2,1,128,1,fp8,fp8,0,0.459881591796875
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,2,1,128,1,float16,fp8,0,0.2516144037246704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,2,2,128,1,float16,float16,0,0.3387808084487915
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,2,2,128,1,float16,fp8,0,0.3282831907272339
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,2,2,128,1,fp8,fp8,0,0.3259632110595703
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,2,1,128,1,float16,float16,0,0.26052639484405515
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,2,1,128,1,fp8,fp8,0,0.25032479763031007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,2,1,128,1,fp8,fp8,0,0.14458880424499512
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,2,2,128,1,float16,float16,0,0.18754880428314208
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,2,2,128,1,float16,fp8,0,0.1804927945137024
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,2,2,128,1,fp8,fp8,0,0.18166400194168092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,2,1,128,1,float16,float16,0,0.14782880544662474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,2,1,128,1,float16,fp8,0,0.14572479724884033
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,2,2,128,1,float16,float16,0,0.1131824016571045
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,2,2,128,1,float16,fp8,0,0.10128159523010254
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,2,2,128,1,fp8,fp8,0,0.1026047945022583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,2,1,128,1,float16,float16,0,0.08910560011863708
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,2,1,128,1,float16,fp8,0,0.08356959819793701
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,2,1,128,1,fp8,fp8,0,0.0840063989162445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,2,2,128,1,float16,float16,0,0.06364480257034302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,2,2,128,1,float16,fp8,0,0.0633135974407196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,2,2,128,1,fp8,fp8,0,0.06315839886665345
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,2,1,128,1,float16,float16,0,0.05573599934577942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,2,1,128,1,float16,fp8,0,0.055846399068832396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,2,1,128,1,fp8,fp8,0,0.05556640028953552
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,2,2,128,1,float16,float16,0,0.048363199830055235
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,2,2,128,1,float16,fp8,0,0.04910559952259064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,2,2,128,1,fp8,fp8,0,0.04930399954319
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,2,1,128,1,float16,float16,0,0.04371039867401123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,2,1,128,1,float16,fp8,0,0.04477440118789673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,2,1,128,1,fp8,fp8,0,0.04433760046958923
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,2,2,128,1,float16,float16,0,0.034620800614356996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,2,2,128,1,float16,fp8,0,0.03671840131282807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,2,2,128,1,fp8,fp8,0,0.03663359880447388
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,2,1,128,1,float16,float16,0,0.034036800265312195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,2,1,128,1,float16,fp8,0,0.03578880131244659
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,2,1,128,1,fp8,fp8,0,0.035913598537445066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,2,1,128,1,float16,float16,0,0.3281519889831543
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,2,1,128,1,float16,fp8,0,0.32003839015960694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,2,1,128,1,fp8,fp8,0,0.3207616090774536
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,2,2,128,1,float16,float16,0,0.24116640090942382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,2,2,128,1,float16,fp8,0,0.23401920795440673
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,2,2,128,1,fp8,fp8,0,0.23480799198150634
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,2,1,128,1,float16,float16,0,0.1817023992538452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,2,1,128,1,float16,fp8,0,0.177183997631073
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,2,1,128,1,fp8,fp8,0,0.17715200185775756
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,2,2,128,1,float16,float16,0,0.1369439959526062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,2,2,128,1,float16,fp8,0,0.13442399501800537
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,2,2,128,1,fp8,fp8,0,0.13447680473327636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,2,1,128,1,float16,float16,0,0.10730719566345215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,2,1,128,1,float16,fp8,0,0.10086400508880615
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,2,1,128,1,fp8,fp8,0,0.09956480264663696
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,2,2,128,1,float16,float16,0,0.08117120265960694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,2,2,128,1,float16,fp8,0,0.0765775978565216
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,2,2,128,1,fp8,fp8,0,0.07595040202140808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,2,1,128,1,float16,float16,0,0.06433280110359192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,2,1,128,1,float16,fp8,0,0.0649616003036499
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,2,1,128,1,fp8,fp8,0,0.06508960127830506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,2,2,128,1,float16,float16,0,0.06207360029220581
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,2,2,128,1,float16,fp8,0,0.06021919846534729
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,2,2,128,1,fp8,fp8,0,0.05934879779815674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,2,1,128,1,float16,float16,0,0.05265759825706482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,2,1,128,1,float16,fp8,0,0.05356159806251526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,2,1,128,1,fp8,fp8,0,0.053748798370361325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,2,2,128,1,float16,float16,0,0.040227198600769044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,2,2,128,1,float16,fp8,0,0.040427199006080626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,2,2,128,1,fp8,fp8,0,0.040355199575424196
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,2,1,128,1,float16,float16,0,0.03599520027637482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,2,1,128,1,float16,fp8,0,0.03764959871768951
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,2,1,128,1,fp8,fp8,0,0.03771199882030487
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,2,2,128,1,float16,float16,0,0.03198240101337433
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,2,2,128,1,float16,fp8,0,0.03219519853591919
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,2,2,128,1,fp8,fp8,0,0.03195520043373108
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,2,1,128,1,float16,float16,0,0.031508800387382505
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,2,1,128,1,float16,fp8,0,0.03172639906406403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,2,1,128,1,fp8,fp8,0,0.03149439990520477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,2,1,128,1,float16,float16,0,0.37575359344482423
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,2,1,128,1,float16,fp8,0,0.3661247968673706
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,2,1,128,1,fp8,fp8,0,0.3686079978942871
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,2,2,128,1,float16,float16,0,0.2828304052352905
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,2,2,128,1,float16,fp8,0,0.27664639949798586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,2,2,128,1,fp8,fp8,0,0.2770623922348022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,2,1,128,1,float16,float16,0,0.20314559936523438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,2,1,128,1,float16,fp8,0,0.20028960704803467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,2,1,128,1,fp8,fp8,0,0.1997439980506897
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,2,2,128,1,float16,float16,0,0.15571680068969726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,2,2,128,1,float16,fp8,0,0.15466400384902954
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,2,2,128,1,fp8,fp8,0,0.15422879457473754
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,2,1,128,1,float16,float16,0,0.11746400594711304
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,2,1,128,1,float16,fp8,0,0.11684479713439941
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,2,1,128,1,fp8,fp8,0,0.11647520065307618
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,2,2,128,1,float16,float16,0,0.051686400175094606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,2,2,128,1,float16,float16,0,0.09282079935073853
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,2,2,128,1,float16,fp8,0,0.08640480041503906
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,2,2,128,1,fp8,fp8,0,0.08558080196380616
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,2,1,128,1,float16,float16,0,0.06951040029525757
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,2,1,128,1,float16,fp8,0,0.06677280068397522
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,2,1,128,1,fp8,fp8,0,0.06754080057144166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,2,2,128,1,float16,fp8,0,0.051815998554229734
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,2,2,128,1,fp8,fp8,0,0.0518127977848053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,2,1,128,1,float16,float16,0,0.04275520145893097
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,2,1,128,1,float16,fp8,0,0.04393759965896606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,2,1,128,1,fp8,fp8,0,0.04402720034122467
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,2,2,128,1,float16,float16,0,0.041761600971221925
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,2,2,128,1,float16,fp8,0,0.04359999895095825
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,2,2,128,1,fp8,fp8,0,0.043350398540496826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,2,1,128,1,float16,float16,0,0.0372191995382309
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,2,1,128,1,float16,fp8,0,0.039022400975227356
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,2,1,128,1,fp8,fp8,0,0.038950398564338684
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,2,2,128,1,float16,float16,0,0.032655999064445496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,2,2,128,1,float16,fp8,0,0.03299840092658997
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,2,2,128,1,fp8,fp8,0,0.03315359950065613
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,2,1,128,1,float16,float16,0,0.03228960037231445
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,2,1,128,1,float16,fp8,0,0.032323199510574344
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,2,1,128,1,fp8,fp8,0,0.03234559893608093
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,2,2,128,1,float16,float16,0,0.02929919958114624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,2,2,128,1,float16,fp8,0,0.031001600623130798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,2,2,128,1,fp8,fp8,0,0.030876800417900085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,2,1,128,1,float16,float16,0,0.029084798693656922
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,2,1,128,1,float16,fp8,0,0.03078399896621704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,2,2,128,1,float16,float16,0,0.24115359783172607
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,2,1,128,1,fp8,fp8,0,0.031196799874305726
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,2,1,128,1,float16,float16,0,0.2990288019180298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,2,1,128,1,float16,fp8,0,0.2947488069534302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,2,1,128,1,fp8,fp8,0,0.2934096097946167
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,2,2,128,1,float16,fp8,0,0.23527839183807372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,2,2,128,1,fp8,fp8,0,0.2363215923309326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,2,1,128,1,float16,float16,0,0.1610111951828003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,2,1,128,1,float16,fp8,0,0.16121280193328857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,2,1,128,1,fp8,fp8,0,0.1611407995223999
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,2,2,128,1,float16,float16,0,0.13451360464096068
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,2,2,128,1,float16,fp8,0,0.1316208004951477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,2,2,128,1,fp8,fp8,0,0.1316640019416809
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,2,1,128,1,float16,float16,0,0.09491680264472961
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,2,1,128,1,float16,fp8,0,0.09317920207977295
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,2,1,128,1,fp8,fp8,0,0.09393919706344604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,2,2,128,1,float16,float16,0,0.07898719906806946
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,2,2,128,1,float16,fp8,0,0.07262240052223205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,2,2,128,1,fp8,fp8,0,0.07090719938278198
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,2,1,128,1,float16,float16,0,0.05444959998130798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,2,1,128,1,float16,fp8,0,0.052799999713897705
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,2,1,128,1,fp8,fp8,0,0.05232160091400147
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,2,2,128,1,float16,float16,0,0.0423088014125824
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,2,2,128,1,float16,fp8,0,0.042895999550819394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,2,2,128,1,fp8,fp8,0,0.04305759966373444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,2,1,128,1,float16,float16,0,0.033855998516082765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,2,1,128,1,float16,fp8,0,0.03529120087623596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,2,2,128,1,float16,float16,0,0.03379839956760407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,2,1,128,1,fp8,fp8,0,0.0355536013841629
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,2,2,128,1,float16,fp8,0,0.03534559905529022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,2,2,128,1,fp8,fp8,0,0.03574880063533783
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,2,1,128,1,float16,float16,0,0.02900480031967163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,2,1,128,1,float16,fp8,0,0.03123840093612671
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,2,1,128,1,fp8,fp8,0,0.030937600135803222
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,2,2,128,1,float16,float16,0,0.02890239953994751
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,2,2,128,1,float16,fp8,0,0.03081600069999695
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,2,2,128,1,fp8,fp8,0,0.03083840012550354
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,2,1,128,1,float16,float16,0,0.02776640057563782
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,2,1,128,1,float16,fp8,0,0.02998400032520294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,2,1,128,1,fp8,fp8,0,0.029692798852920532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,2,2,128,1,float16,float16,0,0.02764959931373596
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,2,2,128,1,float16,fp8,0,0.029657599329948426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,2,2,128,1,fp8,fp8,0,0.029550400376319886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,2,1,128,1,float16,float16,0,0.02766079902648926
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,2,1,128,1,float16,fp8,0,0.029423999786376952
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,2,1,128,1,fp8,fp8,0,0.02916640043258667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,2,2,128,1,float16,float16,0,0.02731040120124817
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,2,2,128,1,float16,fp8,0,0.02924799919128418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,2,2,128,1,fp8,fp8,0,0.029257598519325256
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,2,1,128,1,float16,float16,0,0.02717280089855194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,2,1,128,1,float16,fp8,0,0.028854399919509888
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,2,1,128,1,fp8,fp8,0,0.02930240035057068
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,2,1,128,1,float16,float16,0,0.15240639448165894
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,2,1,128,1,float16,fp8,0,0.1491312026977539
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,2,1,128,1,fp8,fp8,0,0.1503327965736389
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,2,2,128,1,float16,float16,0,0.13033119440078736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,2,2,128,1,float16,fp8,0,0.1233247995376587
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,2,2,128,1,fp8,fp8,0,0.12248320579528808
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,2,1,128,1,float16,float16,0,0.08814719915390015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,2,1,128,1,float16,fp8,0,0.0869983971118927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,2,1,128,1,fp8,fp8,0,0.08695679903030396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,2,2,128,1,float16,float16,0,0.07392320036888123
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,2,2,128,1,float16,fp8,0,0.0651087999343872
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,2,2,128,1,fp8,fp8,0,0.06517599821090699
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,2,1,128,1,float16,float16,0,0.05039200186729431
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,2,1,128,1,float16,fp8,0,0.04732320010662079
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,2,1,128,1,fp8,fp8,0,0.047916799783706665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,2,2,128,1,float16,float16,0,0.039776000380516055
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,2,2,128,1,float16,fp8,0,0.039566400647163394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,2,2,128,1,fp8,fp8,0,0.03938559889793396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,2,1,128,1,float16,float16,0,0.03149600028991699
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,2,1,128,1,float16,fp8,0,0.03264960050582886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,2,1,128,1,fp8,fp8,0,0.03233599960803986
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,2,2,128,1,float16,float16,0,0.03073279857635498
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,2,2,128,1,float16,fp8,0,0.0313616007566452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,2,2,128,1,fp8,fp8,0,0.031891199946403506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,2,1,128,1,float16,float16,0,0.026688000559806822
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,2,1,128,1,float16,fp8,0,0.027353599667549133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,2,1,128,1,fp8,fp8,0,0.02773120105266571
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,2,2,128,1,float16,float16,0,0.026128000020980834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,2,2,128,1,float16,fp8,0,0.027132800221443175
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,2,2,128,1,fp8,fp8,0,0.02714720070362091
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,2,1,128,1,float16,float16,0,0.025251200795173644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,2,1,128,1,float16,fp8,0,0.026451200246810913
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,2,1,128,1,fp8,fp8,0,0.026704001426696777
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,2,2,128,1,float16,float16,0,0.024609600007534028
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,2,2,128,1,float16,fp8,0,0.026017600297927858
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,2,2,128,1,fp8,fp8,0,0.025891199707984924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,2,1,128,1,float16,float16,0,0.02457599937915802
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,2,1,128,1,float16,fp8,0,0.025760000944137572
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,2,1,128,1,fp8,fp8,0,0.02547999918460846
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,2,2,128,1,float16,float16,0,0.024278399348258973
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,2,2,128,1,float16,fp8,0,0.025363200902938844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,2,2,128,1,fp8,fp8,0,0.025342398881912233
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,2,1,128,1,float16,float16,0,0.024379199743270873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,2,1,128,1,float16,fp8,0,0.025435200333595274
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,2,1,128,1,fp8,fp8,0,0.025390401482582092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,2,2,128,1,float16,float16,0,0.022809599339962006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,2,2,128,1,float16,fp8,0,0.024216000735759736
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,2,2,128,1,fp8,fp8,0,0.024259200692176817
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,2,1,128,1,float16,float16,0,0.023124800622463228
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,2,1,128,1,float16,fp8,0,0.024201600253582
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,2,1,128,1,fp8,fp8,0,0.024140800535678863
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,2,1,128,1,float16,float16,0,0.08798080086708068
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,2,1,128,1,float16,fp8,0,0.0840224027633667
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,2,1,128,1,fp8,fp8,0,0.08435999751091003
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,2,2,128,1,float16,float16,0,0.0730512022972107
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,2,2,128,1,float16,fp8,0,0.06236479878425598
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,2,2,128,1,fp8,fp8,0,0.06175839900970459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,2,1,128,1,float16,float16,0,0.04778240025043488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,2,1,128,1,float16,fp8,0,0.04494400024414062
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,2,1,128,1,fp8,fp8,0,0.045388799905776975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,2,2,128,1,float16,float16,0,0.03968639969825745
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,2,2,128,1,float16,fp8,0,0.03827680051326752
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,2,2,128,1,fp8,fp8,0,0.03835839927196503
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,2,1,128,1,float16,float16,0,0.030745598673820495
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,2,1,128,1,float16,fp8,0,0.03152959942817688
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,2,1,128,1,fp8,fp8,0,0.03115360140800476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,2,2,128,1,float16,float16,0,0.02980160117149353
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,2,2,128,1,float16,fp8,0,0.029835200309753417
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,2,2,128,1,fp8,fp8,0,0.030502399802207945
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,2,1,128,1,float16,float16,0,0.025646400451660157
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,2,1,128,1,float16,fp8,0,0.02594879865646362
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,2,1,128,1,fp8,fp8,0,0.02589600086212158
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,2,2,128,1,float16,float16,0,0.024935999512672426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,2,2,128,1,float16,fp8,0,0.025540798902511597
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,2,2,128,1,fp8,fp8,0,0.025510400533676147
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,2,1,128,1,float16,float16,0,0.02350880056619644
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,2,1,128,1,float16,fp8,0,0.02462719976902008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,2,1,128,1,fp8,fp8,0,0.02476000040769577
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,2,2,128,1,float16,float16,0,0.023183999955654143
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,2,2,128,1,float16,fp8,0,0.023937599360942842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,2,2,128,1,fp8,fp8,0,0.02402079999446869
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,2,1,128,1,float16,float16,0,0.02329760044813156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,2,1,128,1,float16,fp8,0,0.023907199501991272
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,2,1,128,1,fp8,fp8,0,0.02391200065612793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,2,2,128,1,float16,float16,0,0.02261119931936264
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,2,2,128,1,float16,fp8,0,0.02338239997625351
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,2,2,128,1,fp8,fp8,0,0.023337599635124207
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,2,1,128,1,float16,float16,0,0.022620800137519836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,2,1,128,1,float16,fp8,0,0.023268799483776092
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,2,1,128,1,fp8,fp8,0,0.022118400037288665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,2,1,128,1,fp8,fp8,0,0.02348479926586151
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,2,2,128,1,float16,float16,0,0.02157599925994873
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,2,2,128,1,float16,fp8,0,0.022268800437450408
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,2,2,128,1,fp8,fp8,0,0.02232159972190857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,2,1,128,1,float16,float16,0,0.021588799357414246
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,2,1,128,1,float16,fp8,0,0.022059200704097746
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,2,2,128,1,float16,float16,0,0.020744000375270844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,2,2,128,1,float16,fp8,0,0.021561600267887115
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,2,2,128,1,fp8,fp8,0,0.021617600321769716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,2,1,128,1,float16,float16,0,0.020502400398254395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,2,1,128,1,float16,fp8,0,0.02146719992160797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,2,1,128,1,fp8,fp8,0,0.02162559926509857
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,2,1,128,1,float16,float16,0,0.050316798686981204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,2,1,128,1,float16,fp8,0,0.047137600183486936
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,2,1,128,1,fp8,fp8,0,0.047646400332450864
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,2,2,128,1,float16,float16,0,0.043094399571418765
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,2,2,128,1,float16,fp8,0,0.043926399946212766
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,2,2,128,1,fp8,fp8,0,0.04355199933052063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,2,1,128,1,float16,float16,0,0.0321152001619339
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,2,1,128,1,float16,fp8,0,0.03226560056209564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,2,1,128,1,fp8,fp8,0,0.03235200047492981
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,2,2,128,1,float16,float16,0,0.029628801345825195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,2,2,128,1,float16,fp8,0,0.030865600705146788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,2,2,128,1,fp8,fp8,0,0.030486398935317995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,2,1,128,1,float16,float16,0,0.026004800200462343
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,2,1,128,1,float16,fp8,0,0.026601600646972656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,2,1,128,1,fp8,fp8,0,0.02632319927215576
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,2,2,128,1,float16,float16,0,0.024854399263858795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,2,2,128,1,float16,fp8,0,0.025681599974632263
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,2,2,128,1,fp8,fp8,0,0.025217598676681517
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,2,1,128,1,float16,float16,0,0.02391200065612793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,2,1,128,1,float16,fp8,0,0.024545599520206452
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,2,1,128,1,fp8,fp8,0,0.025150400400161744
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,2,2,128,1,float16,float16,0,0.023294399678707122
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,2,2,128,1,float16,fp8,0,0.024001599848270418
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,2,2,128,1,fp8,fp8,0,0.024145600199699403
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,2,1,128,1,float16,float16,0,0.02300799936056137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,2,1,128,1,float16,fp8,0,0.02367520034313202
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,2,1,128,1,fp8,fp8,0,0.023825600743293762
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,2,2,128,1,float16,float16,0,0.02268960028886795
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,2,2,128,1,float16,fp8,0,0.02354079931974411
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,2,2,128,1,fp8,fp8,0,0.023452800512313843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,2,1,128,1,float16,float16,0,0.022572800517082214
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,2,1,128,1,float16,fp8,0,0.02345760017633438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,2,1,128,1,fp8,fp8,0,0.021963199973106383
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,2,1,128,1,fp8,fp8,0,0.023289600014686586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,2,2,128,1,float16,float16,0,0.02112479954957962
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,2,2,128,1,float16,fp8,0,0.022096000611782074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,2,2,128,1,fp8,fp8,0,0.021828800439834595
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,2,1,128,1,float16,float16,0,0.02156960070133209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,2,1,128,1,float16,fp8,0,0.022327999770641326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,2,2,128,1,float16,float16,0,0.020579199492931365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,2,2,128,1,float16,fp8,0,0.021147200465202333
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,2,2,128,1,fp8,fp8,0,0.021347199380397797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,2,2,128,1,fp8,fp8,0,0.021652799844741822
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,2,1,128,1,float16,float16,0,0.020496000349521638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,2,1,128,1,float16,fp8,0,0.02173919975757599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,2,1,128,1,fp8,fp8,0,0.02147199958562851
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,2,2,128,1,float16,float16,0,0.02035679966211319
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,2,2,128,1,float16,fp8,0,0.021187199652194975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,2,1,128,1,float16,float16,0,0.02067359983921051
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,2,1,128,1,float16,fp8,0,0.021227200329303742
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,2,1,128,1,fp8,fp8,0,0.021113599836826324
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,2,1,128,1,float16,float16,0,0.03730559945106506
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,2,1,128,1,float16,fp8,0,0.03901439905166626
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,2,1,128,1,fp8,fp8,0,0.03901920020580292
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,2,2,128,1,float16,float16,0,0.03461439907550812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,2,2,128,1,float16,fp8,0,0.036662399768829346
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,2,2,128,1,fp8,fp8,0,0.036499199271202085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,2,1,128,1,float16,float16,0,0.026596799492836
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,2,1,128,1,float16,fp8,0,0.027884799242019653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,2,1,128,1,fp8,fp8,0,0.02736639976501465
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,2,2,128,1,float16,float16,0,0.025934401154518127
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,2,2,128,1,float16,fp8,0,0.02651839852333069
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,2,2,128,1,fp8,fp8,0,0.026203200221061707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,2,1,128,1,float16,float16,0,0.024607999622821806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,2,1,128,1,float16,fp8,0,0.025041601061820982
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,2,1,128,1,fp8,fp8,0,0.025235199928283693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,2,2,128,1,float16,float16,0,0.023502400517463683
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,2,2,128,1,float16,float16,0,0.022675199806690215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,2,2,128,1,float16,fp8,0,0.024868799746036528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,2,2,128,1,fp8,fp8,0,0.024585600197315215
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,2,1,128,1,float16,float16,0,0.02340960055589676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,2,1,128,1,float16,fp8,0,0.024484799802303316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,2,1,128,1,fp8,fp8,0,0.023984000086784363
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,2,2,128,1,float16,fp8,0,0.023481599986553192
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,2,2,128,1,fp8,fp8,0,0.02364960014820099
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,2,1,128,1,float16,float16,0,0.02284799963235855
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,2,1,128,1,float16,fp8,0,0.02361920028924942
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,2,1,128,1,fp8,fp8,0,0.02356639951467514
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,2,2,128,1,float16,float16,0,0.02167679965496063
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,2,2,128,1,float16,fp8,0,0.0222448006272316
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,2,2,128,1,fp8,fp8,0,0.021939200162887574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,2,1,128,1,float16,float16,0,0.021238400042057036
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,2,1,128,1,float16,fp8,0,0.022217600047588347
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,2,1,128,1,fp8,fp8,0,0.022155199944972993
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,2,2,128,1,float16,float16,0,0.020534400641918183
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,2,2,128,1,float16,fp8,0,0.021356800198554994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,2,2,128,1,fp8,fp8,0,0.02138720005750656
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,2,1,128,1,float16,float16,0,0.0203792005777359
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,2,1,128,1,float16,fp8,0,0.0212351992726326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,2,1,128,1,fp8,fp8,0,0.021347199380397797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,2,2,128,1,float16,float16,0,0.020270399749279022
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,2,2,128,1,float16,fp8,0,0.0210207998752594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,2,2,128,1,float16,fp8,0,0.0212336003780365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,2,2,128,1,fp8,fp8,0,0.021356800198554994
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,2,1,128,1,float16,float16,0,0.020337599515914916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,2,1,128,1,float16,fp8,0,0.02120320051908493
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,2,1,128,1,fp8,fp8,0,0.021155199408531188
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,2,2,128,1,float16,float16,0,0.019889600574970245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,2,2,128,1,fp8,fp8,0,0.021246400475502015
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,2,1,128,1,float16,float16,0,0.020103999972343446
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,2,1,128,1,float16,fp8,0,0.021096000075340272
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,2,1,128,1,fp8,fp8,0,0.021011200547218323
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,2,1,128,1,float16,float16,0,0.03341439962387085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,2,1,128,1,float16,fp8,0,0.03489919900894165
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,2,1,128,1,fp8,fp8,0,0.034815999865531924
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,2,2,128,1,float16,float16,0,0.030103999376296996
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,2,2,128,1,float16,fp8,0,0.032278400659561154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,2,2,128,1,fp8,fp8,0,0.031667199730873105
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,2,1,128,1,float16,float16,0,0.026348799467086792
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,2,1,128,1,float16,fp8,0,0.026791998744010927
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,2,1,128,1,fp8,fp8,0,0.026977598667144775
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,2,2,128,1,float16,float16,0,0.02468319982290268
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,2,2,128,1,float16,fp8,0,0.02435680031776428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,2,2,128,1,float16,fp8,0,0.02518239915370941
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,2,2,128,1,fp8,fp8,0,0.025198400020599365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,2,1,128,1,float16,float16,0,0.024187199771404266
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,2,1,128,1,float16,fp8,0,0.024743999540805816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,2,1,128,1,fp8,fp8,0,0.02528800070285797
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,2,2,128,1,float16,float16,0,0.023468799889087677
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,2,2,128,1,fp8,fp8,0,0.02434239983558655
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,2,1,128,1,float16,float16,0,0.02345760017633438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,2,1,128,1,float16,fp8,0,0.024120000004768372
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,2,1,128,1,fp8,fp8,0,0.024198399484157564
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,2,2,128,1,float16,float16,0,0.02168000042438507
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,2,2,128,1,float16,fp8,0,0.02272160053253174
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,2,2,128,1,fp8,fp8,0,0.022491200268268584
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,2,1,128,1,float16,float16,0,0.021484799683094025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,2,1,128,1,float16,fp8,0,0.022243200242519377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,2,1,128,1,fp8,fp8,0,0.02234880030155182
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,2,2,128,1,float16,float16,0,0.020947200059890748
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,2,2,128,1,float16,fp8,0,0.02114879935979843
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,2,2,128,1,float16,fp8,0,0.021326400339603424
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,2,2,128,1,fp8,fp8,0,0.02154400050640106
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,2,1,128,1,float16,float16,0,0.020619200170040132
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,2,1,128,1,float16,fp8,0,0.02152799963951111
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,2,1,128,1,fp8,fp8,0,0.02154559940099716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,2,2,128,1,float16,float16,0,0.02051360011100769
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,2,2,128,1,fp8,fp8,0,0.020971199870109557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,2,1,128,1,float16,float16,0,0.020336000621318816
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,2,1,128,1,float16,fp8,0,0.020878399908542632
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,2,1,128,1,fp8,fp8,0,0.021108800172805788
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,2,2,128,1,float16,float16,0,0.020078399777412416
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,2,2,128,1,float16,fp8,0,0.0212336003780365
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,2,2,128,1,fp8,fp8,0,0.021166400611400606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,2,1,128,1,float16,float16,0,0.020134399831295013
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,2,1,128,1,float16,fp8,0,0.021076799929142
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,2,1,128,1,fp8,fp8,0,0.02101919949054718
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,2,2,128,1,float16,float16,0,0.0200655996799469
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,2,2,128,1,float16,fp8,0,0.020945599675178526
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,2,2,128,1,fp8,fp8,0,0.021129600703716278
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,2,1,128,1,float16,float16,0,0.01983039975166321
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,2,1,128,1,float16,fp8,0,0.02102400064468384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,2,1,128,1,fp8,fp8,0,0.02104640007019043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,1,1,128,1,float16,float16,0,0.5178336143493653
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,1,1,128,1,float16,fp8,0,0.5006912231445313
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,1,1,128,1,float16,float16,0,0.2683919906616211
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,1,1,128,1,fp8,fp8,0,0.5034751892089844
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,1,1,128,1,float16,fp8,0,0.25636000633239747
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,1,1,128,1,float16,fp8,0,0.36263360977172854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,1,1,128,1,fp8,fp8,0,0.2557919979095459
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,1,1,128,1,float16,float16,0,0.2373215913772583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,1,1,128,1,float16,fp8,0,0.23566079139709473
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,1,1,128,1,fp8,fp8,0,0.23140161037445067
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,1,1,128,1,float16,float16,0,0.3731872081756592
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,1,1,128,1,float16,float16,0,0.21104159355163574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,1,1,128,1,fp8,fp8,0,0.3615231990814209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,1,1,128,1,float16,fp8,0,0.19861760139465331
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,1,1,128,1,fp8,fp8,0,0.19788000583648682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,1,1,128,1,float16,float16,0,0.12254559993743896
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,1,1,128,1,float16,fp8,0,0.12017279863357544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,1,1,128,1,fp8,fp8,0,0.11804800033569336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,1,1,128,1,float16,float16,0,0.2972831964492798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,1,1,128,1,float16,fp8,0,0.28740320205688474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,1,1,128,1,float16,fp8,0,0.10059839487075806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,1,1,128,1,fp8,fp8,0,0.2874367952346802
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,1,1,128,1,float16,float16,0,0.17653119564056396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,1,1,128,1,float16,fp8,0,0.1670896053314209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,1,1,128,1,fp8,fp8,0,0.16684319972991943
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,1,1,128,1,fp8,fp8,0,0.3572016000747681
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,1,1,128,1,float16,float16,0,0.1049504041671753
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,1,1,128,1,fp8,fp8,0,0.1002351999282837
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,1,1,128,1,float16,float16,0,0.36934239864349366
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,1,1,128,1,float16,fp8,0,0.35647199153900144
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,1,1,128,1,float16,float16,0,0.22431681156158448
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,1,1,128,1,float16,fp8,0,0.21196799278259276
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,1,1,128,1,fp8,fp8,0,0.20964479446411133
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,1,1,128,1,float16,float16,0,0.1397104024887085
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,1,1,128,1,float16,fp8,0,0.14010080099105834
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,1,1,128,1,fp8,fp8,0,0.1380784034729004
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,1,1,128,1,float16,float16,0,0.08512639999389648
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,1,1,128,1,float16,fp8,0,0.08564000129699707
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,1,1,128,1,fp8,fp8,0,0.08650079965591431
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,1,1,128,1,float16,float16,0,0.2490992069244385
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,1,1,128,1,float16,fp8,0,0.23998079299926758
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,1,1,128,1,fp8,fp8,0,0.23950719833374023
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,1,1,128,1,float16,float16,0,0.14631199836730957
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,1,1,128,1,float16,fp8,0,0.13579200506210326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,1,1,128,1,fp8,fp8,0,0.13504960536956787
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,1,1,128,1,float16,float16,0,0.09231039881706238
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,1,1,128,1,float16,fp8,0,0.09076160192489624
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,1,1,128,1,fp8,fp8,0,0.08963840007781983
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,1,1,128,1,float16,float16,0,0.05720319747924805
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,1,1,128,1,float16,fp8,0,0.057555198669433594
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,1,1,128,1,fp8,fp8,0,0.05803040266036987
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,1,1,128,1,float16,float16,0,0.25206880569458007
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,1,1,128,1,float16,fp8,0,0.2457632064819336
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,1,1,128,1,fp8,fp8,0,0.24308960437774657
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,1,1,128,1,float16,float16,0,0.15301120281219482
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,1,1,128,1,float16,fp8,0,0.14167840480804444
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,1,1,128,1,fp8,fp8,0,0.1426751971244812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,1,1,128,1,float16,float16,0,0.08895519971847535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,1,1,128,1,float16,fp8,0,0.08909279704093934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,1,1,128,1,float16,fp8,0,0.04241119921207428
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,1,1,128,1,fp8,fp8,0,0.08842560052871704
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,1,1,128,1,float16,float16,0,0.06093599796295166
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,1,1,128,1,float16,fp8,0,0.06043999791145325
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,1,1,128,1,fp8,fp8,0,0.06087520122528076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,1,1,128,1,float16,float16,0,0.04106720089912415
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,1,1,128,1,fp8,fp8,0,0.04215520024299622
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,1,1,128,1,fp8,fp8,0,0.10233919620513916
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,1,1,128,1,float16,float16,0,0.17893439531326294
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,1,1,128,1,float16,fp8,0,0.17270400524139404
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,1,1,128,1,fp8,fp8,0,0.17067840099334716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,1,1,128,1,float16,float16,0,0.10642399787902831
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,1,1,128,1,float16,fp8,0,0.10056320428848267
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,1,1,128,1,float16,float16,0,0.07456160187721253
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,1,1,128,1,float16,fp8,0,0.03557760119438171
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,1,1,128,1,float16,fp8,0,0.07167519927024842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,1,1,128,1,fp8,fp8,0,0.07260640263557434
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,1,1,128,1,float16,float16,0,0.04514720141887665
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,1,1,128,1,float16,fp8,0,0.04563519954681396
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,1,1,128,1,fp8,fp8,0,0.04591839909553528
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,1,1,128,1,float16,float16,0,0.035585600137710574
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,1,1,128,1,fp8,fp8,0,0.03556160032749176
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,1,1,128,1,float16,float16,0,0.19032640457153321
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,1,1,128,1,float16,fp8,0,0.18398239612579345
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,1,1,128,1,fp8,fp8,0,0.18456640243530273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,1,1,128,1,float16,float16,0,0.11472159624099731
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,1,1,128,1,float16,fp8,0,0.10393120050430298
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,1,1,128,1,fp8,fp8,0,0.10319839715957642
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,1,1,128,1,float16,float16,0,0.06476320028305053
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,1,1,128,1,float16,fp8,0,0.06291199922561645
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,1,1,128,1,fp8,fp8,0,0.06305279731750488
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,1,1,128,1,float16,float16,0,0.047907200455665586
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,1,1,128,1,float16,fp8,0,0.04905279874801636
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,1,1,128,1,fp8,fp8,0,0.04907839894294739
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,1,1,128,1,float16,float16,0,0.0347791999578476
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,1,1,128,1,float16,fp8,0,0.036683198809623715
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,1,1,128,1,fp8,fp8,0,0.036355200409889224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,1,1,128,1,float16,float16,0,0.031220799684524535
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,1,1,128,1,float16,fp8,0,0.03150720000267029
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,1,1,128,1,fp8,fp8,0,0.03148959875106812
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,1,1,128,1,float16,float16,0,0.13938239812850953
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,1,1,128,1,float16,fp8,0,0.13515520095825195
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,1,1,128,1,fp8,fp8,0,0.1352463960647583
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,1,1,128,1,float16,float16,0,0.08374879956245422
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,1,1,128,1,fp8,fp8,0,0.05999040007591248
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,1,1,128,1,float16,fp8,0,0.07663840055465698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,1,1,128,1,fp8,fp8,0,0.07623999714851379
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,1,1,128,1,float16,fp8,0,0.040038400888442995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,1,1,128,1,float16,float16,0,0.06244159936904907
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,1,1,128,1,float16,fp8,0,0.06061440110206604
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,1,1,128,1,float16,float16,0,0.04019039869308472
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,1,1,128,1,fp8,fp8,0,0.04012160003185272
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,1,1,128,1,float16,float16,0,0.031948798894882204
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,1,1,128,1,float16,fp8,0,0.03194240033626557
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,1,1,128,1,fp8,fp8,0,0.03207359910011291
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,1,1,128,1,float16,float16,0,0.029598399996757507
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,1,1,128,1,fp8,fp8,0,0.15627199411392212
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,1,1,128,1,float16,fp8,0,0.031094399094581605
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,1,1,128,1,fp8,fp8,0,0.031148800253868104
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,1,1,128,1,float16,float16,0,0.15856319665908813
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,1,1,128,1,float16,fp8,0,0.1554255962371826
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,1,1,128,1,float16,float16,0,0.0940783977508545
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,1,1,128,1,float16,fp8,0,0.08888480067253113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,1,1,128,1,fp8,fp8,0,0.08723679780960084
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,1,1,128,1,float16,float16,0,0.052046400308609006
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,1,1,128,1,float16,fp8,0,0.051395201683044435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,1,1,128,1,fp8,fp8,0,0.051708799600601194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,1,1,128,1,float16,float16,0,0.041731199622154234
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,1,1,128,1,float16,fp8,0,0.043315199017524716
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,1,1,128,1,fp8,fp8,0,0.04306240081787109
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,1,1,128,1,float16,float16,0,0.0328575998544693
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,1,1,128,1,float16,fp8,0,0.033048000931739804
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,1,1,128,1,fp8,fp8,0,0.033339199423789975
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,1,1,128,1,float16,float16,0,0.029371199011802674
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,1,1,128,1,float16,fp8,0,0.030969598889350893
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,1,1,128,1,fp8,fp8,0,0.031139200925827025
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,1,1,128,1,float16,float16,0,0.028692799806594848
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,1,1,128,1,float16,fp8,0,0.029841598868370057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,1,1,128,1,fp8,fp8,0,0.02996160089969635
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,1,1,128,1,float16,float16,0,0.1347584009170532
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,1,1,128,1,float16,fp8,0,0.13362720012664794
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,1,1,128,1,fp8,fp8,0,0.13287359476089478
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,1,1,128,1,float16,float16,0,0.07973279953002929
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,1,1,128,1,float16,fp8,0,0.073852801322937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,1,1,128,1,fp8,fp8,0,0.07260159850120544
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,1,1,128,1,float16,float16,0,0.04244320094585419
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,1,1,128,1,float16,fp8,0,0.04346719980239868
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,1,1,128,1,fp8,fp8,0,0.04323360025882721
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,1,1,128,1,float16,float16,0,0.033934399485588074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,1,1,128,1,float16,fp8,0,0.035420799255371095
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,1,1,128,1,fp8,fp8,0,0.035471999645233156
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,1,1,128,1,float16,float16,0,0.029465600848197937
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,1,1,128,1,float16,fp8,0,0.03084160089492798
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,1,1,128,1,fp8,fp8,0,0.030795198678970338
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,1,1,128,1,float16,float16,0,0.028196799755096435
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,1,1,128,1,float16,fp8,0,0.029971200227737426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,1,1,128,1,fp8,fp8,0,0.029819199442863466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,1,1,128,1,float16,float16,0,0.027235201001167296
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,1,1,128,1,float16,fp8,0,0.02916319966316223
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,1,1,128,1,fp8,fp8,0,0.029529601335525513
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,1,1,128,1,float16,float16,0,0.026923200488090514
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,1,1,128,1,float16,fp8,0,0.02951200008392334
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,1,1,128,1,fp8,fp8,0,0.029660800099372865
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,1,1,128,1,float16,float16,0,0.07480159997940064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,1,1,128,1,float16,fp8,0,0.0680288016796112
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,1,1,128,1,fp8,fp8,0,0.06634719967842102
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,1,1,128,1,float16,float16,0,0.041038399934768675
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,1,1,128,1,float16,fp8,0,0.0406464010477066
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,1,1,128,1,fp8,fp8,0,0.040540799498558044
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,1,1,128,1,float16,float16,0,0.031137600541114807
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,1,1,128,1,float16,fp8,0,0.03262239992618561
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,1,1,128,1,fp8,fp8,0,0.03254559934139252
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,1,1,128,1,float16,float16,0,0.02621760070323944
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,1,1,128,1,float16,fp8,0,0.027775999903678895
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,1,1,128,1,fp8,fp8,0,0.027657601237297057
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,1,1,128,1,float16,float16,0,0.024903999269008638
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,1,1,128,1,float16,fp8,0,0.026177600026130676
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,1,1,128,1,fp8,fp8,0,0.026027199625968934
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,1,1,128,1,float16,float16,0,0.024459199607372285
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,1,1,128,1,float16,fp8,0,0.02550399899482727
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,1,1,128,1,fp8,fp8,0,0.025436800718307496
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,1,1,128,1,float16,float16,0,0.024115200340747833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,1,1,128,1,float16,fp8,0,0.025297600030899047
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,1,1,128,1,float16,fp8,0,0.04003680050373078
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,1,1,128,1,fp8,fp8,0,0.025563201308250426
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,1,1,128,1,float16,float16,0,0.022702400386333466
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,1,1,128,1,float16,fp8,0,0.024040000140666963
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,1,1,128,1,fp8,fp8,0,0.02428639978170395
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,1,1,128,1,float16,float16,0,0.04027040004730224
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,1,1,128,1,fp8,fp8,0,0.03937920033931732
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,1,1,128,1,float16,float16,0,0.030516800284385682
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,1,1,128,1,float16,fp8,0,0.030878400802612303
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,1,1,128,1,fp8,fp8,0,0.03127039968967438
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,1,1,128,1,float16,float16,0,0.025022399425506592
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,1,1,128,1,float16,fp8,0,0.025761601328849793
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,1,1,128,1,fp8,fp8,0,0.025923201441764833
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,1,1,128,1,float16,float16,0,0.02356960028409958
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,1,1,128,1,float16,fp8,0,0.024135999381542206
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,1,1,128,1,fp8,fp8,0,0.02421119958162308
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,1,1,128,1,float16,float16,0,0.022793599963188173
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,1,1,128,1,float16,fp8,0,0.023464000225067137
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,1,1,128,1,float16,float16,0,0.022700800001621245
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,1,1,128,1,fp8,fp8,0,0.02335200011730194
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,1,1,128,1,float16,fp8,0,0.023358400166034698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,1,1,128,1,fp8,fp8,0,0.023369599878787995
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,1,1,128,1,float16,float16,0,0.02141599953174591
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,1,1,128,1,float16,fp8,0,0.022324800491333008
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,1,1,128,1,fp8,fp8,0,0.022100800275802614
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,1,1,128,1,float16,float16,0,0.020720000565052032
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,1,1,128,1,float16,fp8,0,0.021400000154972076
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,1,1,128,1,fp8,fp8,0,0.021547199785709382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,1,1,128,1,float16,float16,0,0.03131679892539978
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,1,1,128,1,float16,fp8,0,0.0318448007106781
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,1,1,128,1,fp8,fp8,0,0.03212479948997497
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,1,1,128,1,float16,float16,0,0.02515999972820282
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,1,1,128,1,float16,fp8,0,0.02627519965171814
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,1,1,128,1,fp8,fp8,0,0.025817599892616273
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,1,1,128,1,float16,float16,0,0.023643200099468232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,1,1,128,1,float16,fp8,0,0.024600000679492952
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,1,1,128,1,fp8,fp8,0,0.024214400351047514
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,1,1,128,1,float16,float16,0,0.022716799378395082
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,1,1,128,1,float16,fp8,0,0.0234607994556427
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,1,1,128,1,fp8,fp8,0,0.023585599660873414
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,1,1,128,1,float16,float16,0,0.02237119972705841
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,1,1,128,1,float16,fp8,0,0.023217600584030152
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,1,1,128,1,fp8,fp8,0,0.02316959947347641
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,1,1,128,1,float16,float16,0,0.021070399880409242
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,1,1,128,1,float16,fp8,0,0.021960000693798064
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,1,1,128,1,fp8,fp8,0,0.02173919975757599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,1,1,128,1,float16,float16,0,0.020340800285339355
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,1,1,128,1,float16,fp8,0,0.02104319930076599
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,1,1,128,1,fp8,fp8,0,0.021073600649833678
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,1,1,128,1,float16,float16,0,0.020278400182723998
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,1,1,128,1,float16,fp8,0,0.021134400367736818
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,1,1,128,1,fp8,fp8,0,0.02131199985742569
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,1,1,128,1,float16,float16,0,0.026649600267410277
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,1,1,128,1,float16,fp8,0,0.027689599990844728
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,1,1,128,1,fp8,fp8,0,0.027350398898124694
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,1,1,128,1,float16,float16,0,0.02447039932012558
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,1,1,128,1,float16,fp8,0,0.025352001190185547
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,1,1,128,1,fp8,fp8,0,0.025164800882339477
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,1,1,128,1,float16,float16,0,0.023211200535297394
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,1,1,128,1,float16,fp8,0,0.02412640005350113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,1,1,128,1,fp8,fp8,0,0.024255999922752382
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,1,1,128,1,float16,float16,0,0.022327999770641326
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,1,1,128,1,float16,fp8,0,0.02351360023021698
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,1,1,128,1,fp8,fp8,0,0.02340639978647232
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,1,1,128,1,float16,float16,0,0.02114560008049011
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,1,1,128,1,float16,fp8,0,0.022174400091171265
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,1,1,128,1,fp8,fp8,0,0.022008000314235686
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,1,1,128,1,float16,float16,0,0.020209600031375886
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,1,1,128,1,float16,fp8,0,0.02131360024213791
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,1,1,128,1,fp8,fp8,0,0.02099999934434891
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,1,1,128,1,float16,float16,0,0.02011200040578842
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,1,1,128,1,float16,fp8,0,0.02102400064468384
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,1,1,128,1,fp8,fp8,0,0.02104640007019043
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,1,1,128,1,float16,float16,0,0.019886399805545806
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,1,1,128,1,float16,fp8,0,0.021007999777793884
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,1,1,128,1,fp8,fp8,0,0.020916800200939178
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,1,1,128,1,float16,float16,0,0.02614560127258301
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,1,1,128,1,float16,fp8,0,0.026366400718688964
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,1,1,128,1,fp8,fp8,0,0.026929599046707154
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,1,1,128,1,float16,float16,0,0.023932799696922302
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,1,1,128,1,float16,fp8,0,0.024952000379562377
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,1,1,128,1,fp8,fp8,0,0.02504960000514984
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,1,1,128,1,float16,float16,0,0.023361599445343016
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,1,1,128,1,float16,fp8,0,0.02418559938669205
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,1,1,128,1,fp8,fp8,0,0.024289600551128387
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,1,1,128,1,float16,float16,0,0.021377600729465485
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,1,1,128,1,float16,fp8,0,0.02216479927301407
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,1,1,128,1,fp8,fp8,0,0.02091359943151474
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,1,1,128,1,fp8,fp8,0,0.022198399901390074
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,1,1,128,1,float16,float16,0,0.020449599623680113
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,1,1,128,1,float16,fp8,0,0.021454399824142455
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,1,1,128,1,fp8,fp8,0,0.02156960070133209
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,1,1,128,1,float16,float16,0,0.02000479996204376
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,1,1,128,1,float16,fp8,0,0.021166400611400606
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,1,1,128,1,float16,float16,0,0.019758400321006776
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,1,1,128,1,float16,fp8,0,0.020982399582862854
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,1,1,128,1,fp8,fp8,0,0.021060800552368163
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,1,1,128,1,float16,float16,0,0.02001280039548874
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,1,1,128,1,float16,fp8,0,0.020873600244522096
SGLang,0.5.6.post2,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,1,1,128,1,fp8,fp8,0,0.021187199652194975
