framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,4,128,1,float16,fp8,0,55.21160380045573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,1,128,1,float16,fp8,0,54.28173828125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,8,128,1,float16,fp8,0,55.473602294921875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,4,128,1,float16,float16,0,60.73554992675781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,1,128,1,float16,float16,0,60.35065714518229
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,2,128,1,float16,float16,0,60.32525634765625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,2,128,1,float16,fp8,0,54.43592834472656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,64,128,1,float16,fp8,0,33.47308858235677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,1,128,1,float16,fp8,0,28.215647379557293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,64,128,1,float16,float16,0,37.20141347249349
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,1,128,1,float16,float16,0,36.55254872639974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,2,128,1,float16,fp8,0,28.73370107014974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,2,128,1,float16,float16,0,34.77776082356771
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,4,128,1,float16,float16,0,36.9249267578125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,64,128,1,float16,fp8,0,14.487706502278646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,64,128,1,float16,float16,0,16.396624247233074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,8,128,1,float16,float16,0,60.34920756022135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,1,128,1,float16,float16,0,15.113077799479166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,1,128,1,float16,fp8,0,13.654794057210287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,4,128,1,float16,fp8,0,27.702662150065105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,8,128,1,float16,float16,0,29.67285919189453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,8,128,1,float16,fp8,0,27.82103983561198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,2,128,1,float16,float16,0,16.449386596679688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,2,128,1,float16,fp8,0,13.71612803141276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,4,128,1,float16,fp8,0,13.904186248779297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,64,128,1,float16,float16,0,7.711802800496419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,4,128,1,float16,float16,0,18.657663981119793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,64,128,1,float16,fp8,0,7.26356824239095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,8,128,1,float16,float16,0,15.95187250773112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,1,128,1,float16,fp8,0,6.873360315958659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,1,128,1,float16,float16,0,7.540826797485352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,8,128,1,float16,fp8,0,13.964730580647787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,2,128,1,float16,float16,0,7.360586802164714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,2,128,1,float16,fp8,0,6.895354588826497
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,4,128,1,float16,fp8,0,6.981903711954753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,4,128,1,float16,float16,0,7.666421254475911
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,8,128,1,float16,float16,0,7.335472106933594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,8,128,1,float16,fp8,0,7.056271870930989
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,1,128,1,float16,fp8,0,32.21507771809896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,1,128,1,float16,float16,0,37.3812255859375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,2,128,1,float16,float16,0,37.97815958658854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,2,128,1,float16,fp8,0,31.190330505371094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,4,128,1,float16,fp8,0,31.97814432779948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,8,128,1,float16,fp8,0,32.20775349934896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,4,128,1,float16,float16,0,36.92797342936198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,8,128,1,float16,float16,0,35.062426249186196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,64,128,1,float16,float16,0,21.119679768880207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,64,128,1,float16,fp8,0,16.82876205444336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,1,128,1,float16,float16,0,17.36689631144206
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,1,128,1,float16,fp8,0,15.587226867675781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,2,128,1,float16,fp8,0,16.377349853515625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,2,128,1,float16,float16,0,17.918368021647137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,4,128,1,float16,fp8,0,16.224756876627605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,4,128,1,float16,float16,0,18.12893295288086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,64,128,1,float16,fp8,0,8.42364247639974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,64,128,1,float16,float16,0,9.427167892456055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,1,128,1,float16,fp8,0,7.936437606811523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,1,128,1,float16,float16,0,9.37118403116862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,2,128,1,float16,float16,0,9.093648274739584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,2,128,1,float16,fp8,0,8.01249631245931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,8,128,1,float16,fp8,0,16.041205088297527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,8,128,1,float16,float16,0,17.440255482991535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,4,128,1,float16,fp8,0,8.032384236653646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,64,128,1,float16,fp8,0,4.293285369873047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,64,128,1,float16,float16,0,4.453392028808594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,4,128,1,float16,float16,0,9.417941411336264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,1,128,1,float16,float16,0,4.034101486206055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,8,128,1,float16,fp8,0,8.075125376383463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,8,128,1,float16,float16,0,9.679205576578775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,1,128,1,float16,fp8,0,4.091562589009603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,2,128,1,float16,float16,0,4.251248041788737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,2,128,1,float16,fp8,0,3.9443947474161782
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,4,128,1,float16,float16,0,4.296778678894043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,4,128,1,float16,fp8,0,4.2348480224609375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,8,128,1,float16,float16,0,4.298314730326335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,8,128,1,float16,fp8,0,4.067546526590983
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,1,128,1,float16,fp8,0,21.935274759928387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,1,128,1,float16,float16,0,26.00457000732422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,4,128,1,float16,fp8,0,22.450393676757812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,2,128,1,float16,fp8,0,22.01148732503255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,8,128,1,float16,fp8,0,22.563130696614582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,4,128,1,float16,float16,0,24.58940887451172
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,2,128,1,float16,float16,0,24.252212524414062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,8,128,1,float16,float16,0,24.02275848388672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,64,128,1,float16,float16,0,13.200602213541666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,1,128,1,float16,fp8,0,11.003456115722656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,64,128,1,float16,fp8,0,12.099695841471354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,1,128,1,float16,float16,0,12.490352630615234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,2,128,1,float16,fp8,0,11.083168029785156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,2,128,1,float16,float16,0,12.636698404947916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,4,128,1,float16,float16,0,12.422592163085938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,4,128,1,float16,fp8,0,11.302591959635416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,64,128,1,float16,fp8,0,6.115936279296875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,64,128,1,float16,float16,0,6.683813095092773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,1,128,1,float16,float16,0,5.6600907643636065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,1,128,1,float16,fp8,0,5.903296152750651
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,2,128,1,float16,float16,0,6.079231897989909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,2,128,1,float16,fp8,0,5.553242365519206
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,8,128,1,float16,float16,0,12.31427256266276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,8,128,1,float16,fp8,0,11.322218577067057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,64,128,1,float16,float16,0,3.104288101196289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,64,128,1,float16,fp8,0,3.069845199584961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,4,128,1,float16,float16,0,5.862490971883138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,8,128,1,float16,fp8,0,5.711141586303711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,8,128,1,float16,float16,0,5.852687835693359
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,1,128,1,float16,float16,0,2.888538678487142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,4,128,1,float16,fp8,0,5.902042388916016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,1,128,1,float16,fp8,0,2.8544480005900064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,2,128,1,float16,float16,0,2.886927922566732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,2,128,1,float16,fp8,0,3.185567855834961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,4,128,1,float16,float16,0,2.916954676310221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,8,128,1,float16,float16,0,2.839797337849935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,4,128,1,float16,fp8,0,3.069727897644043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,8,128,1,float16,fp8,0,2.909424146016439
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,1,128,1,float16,fp8,0,28.59827168782552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,2,128,1,float16,fp8,0,28.741322835286457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,1,128,1,float16,float16,0,32.335652669270836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,4,128,1,float16,fp8,0,29.530309041341145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,2,128,1,float16,float16,0,32.40190887451172
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,4,128,1,float16,float16,0,33.82267761230469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,8,128,1,float16,float16,0,32.020965576171875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,8,128,1,float16,fp8,0,29.44707743326823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,1,128,1,float16,float16,0,16.119915008544922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,64,128,1,float16,float16,0,17.649300893147785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,64,128,1,float16,fp8,0,15.941808064778646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,1,128,1,float16,fp8,0,14.333904266357422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,2,128,1,float16,float16,0,15.72817611694336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,2,128,1,float16,fp8,0,14.40765380859375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,4,128,1,float16,float16,0,17.306101481119793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,1,128,1,float16,float16,0,7.80998420715332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,64,128,1,float16,fp8,0,8.004938761393229
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,64,128,1,float16,float16,0,8.956799825032553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,1,128,1,float16,fp8,0,7.1690934499104815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,4,128,1,float16,fp8,0,14.810490926106771
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,2,128,1,float16,float16,0,8.094330469767252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,8,128,1,float16,float16,0,16.642079671223957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,8,128,1,float16,fp8,0,14.789744059244791
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,2,128,1,float16,fp8,0,7.8147837320963545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,4,128,1,float16,float16,0,8.049519856770834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,64,128,1,float16,float16,0,4.621813456217448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,4,128,1,float16,fp8,0,7.716442743937175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,8,128,1,float16,float16,0,8.456181208292643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,1,128,1,float16,float16,0,3.659855842590332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,8,128,1,float16,fp8,0,7.453280131022136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,64,128,1,float16,fp8,0,4.628922780354817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,1,128,1,float16,fp8,0,3.741642634073893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,2,128,1,float16,float16,0,3.978666623433431
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,2,128,1,float16,fp8,0,3.7140585581461587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,4,128,1,float16,float16,0,3.835882822672526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,64,128,1,float16,float16,0,2.0580053329467773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,4,128,1,float16,fp8,0,4.033925374348958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,8,128,1,float16,float16,0,3.820821444193522
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,64,128,1,float16,fp8,0,2.1255359649658203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,8,128,1,float16,fp8,0,3.7230666478474936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,1,128,1,float16,float16,0,1.9140532811482747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,1,128,1,float16,fp8,0,1.8592160542805989
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,2,128,1,float16,float16,0,1.8988800048828125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,4,128,1,float16,float16,0,1.9072693188985188
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,2,128,1,float16,fp8,0,1.9023680686950684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,8,128,1,float16,float16,0,1.8304160435994465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,4,128,1,float16,fp8,0,2.163856029510498
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,8,128,1,float16,fp8,0,1.902133305867513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,1,128,1,float16,float16,0,18.847530364990234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,1,128,1,float16,fp8,0,16.656117757161457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,2,128,1,float16,float16,0,18.787823994954426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,2,128,1,float16,fp8,0,16.759204864501953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,4,128,1,float16,fp8,0,17.58735529581706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,8,128,1,float16,fp8,0,17.24778111775716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,8,128,1,float16,float16,0,19.356751759847004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,4,128,1,float16,float16,0,18.62280527750651
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,1,128,1,float16,fp8,0,8.346015930175781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,1,128,1,float16,float16,0,9.065301259358725
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,2,128,1,float16,float16,0,9.378426869710287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,64,128,1,float16,fp8,0,9.562581380208334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,64,128,1,float16,float16,0,9.805178960164389
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,2,128,1,float16,fp8,0,8.373845418294271
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,4,128,1,float16,float16,0,9.085082372029623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,4,128,1,float16,fp8,0,8.703397115071615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,64,128,1,float16,float16,0,4.939237276713054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,64,128,1,float16,fp8,0,4.797151883443196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,1,128,1,float16,float16,0,4.28055477142334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,1,128,1,float16,fp8,0,4.159680048624675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,2,128,1,float16,fp8,0,4.183024088541667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,2,128,1,float16,float16,0,4.6226240793863935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,8,128,1,float16,float16,0,9.861978530883789
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,8,128,1,float16,fp8,0,8.828885396321615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,64,128,1,float16,float16,0,2.4143786430358887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,4,128,1,float16,float16,0,4.347514788309733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,64,128,1,float16,fp8,0,2.561391989390055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,4,128,1,float16,fp8,0,4.364997227986653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,8,128,1,float16,float16,0,4.373141288757324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,8,128,1,float16,fp8,0,4.336208025614421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,1,128,1,float16,float16,0,2.4641332626342773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,1,128,1,float16,fp8,0,2.1336159706115723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,2,128,1,float16,float16,0,2.1328694025675454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,2,128,1,float16,fp8,0,2.3043999671936035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,4,128,1,float16,fp8,0,2.213749408721924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,4,128,1,float16,float16,0,2.2036587397257485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,64,128,1,float16,float16,0,1.2505546410878499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,8,128,1,float16,float16,0,2.158261299133301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,64,128,1,float16,fp8,0,1.2563680013020833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,1,128,1,float16,float16,0,1.076368014017741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,8,128,1,float16,fp8,0,2.2012106577555337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,1,128,1,float16,fp8,0,1.1689066886901855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,2,128,1,float16,float16,0,1.10425599416097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,2,128,1,float16,fp8,0,1.126810630162557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,4,128,1,float16,float16,0,1.1194079717000325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,4,128,1,float16,fp8,0,1.1355626583099365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,8,128,1,float16,float16,0,1.0888480345408122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,8,128,1,float16,fp8,0,1.1322346528371174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,2,128,1,float16,fp8,0,15.907515207926432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,1,128,1,float16,fp8,0,15.892512003580729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,2,128,1,float16,float16,0,17.12130610148112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,4,128,1,float16,float16,0,17.98315684000651
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,1,128,1,float16,float16,0,16.955615997314453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,4,128,1,float16,fp8,0,16.886810302734375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,8,128,1,float16,float16,0,17.60468292236328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,1,128,1,float16,float16,0,8.497317632039389
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,1,128,1,float16,fp8,0,7.905567804972331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,2,128,1,float16,float16,0,8.333936055501303
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,2,128,1,float16,fp8,0,7.901114781697591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,64,128,1,float16,float16,0,10.33463986714681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,64,128,1,float16,fp8,0,9.509077072143555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,4,128,1,float16,float16,0,8.68006960550944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,8,128,1,float16,fp8,0,16.476783752441406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,1,128,1,float16,float16,0,3.9235092798868814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,64,128,1,float16,float16,0,4.7930558522542315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,1,128,1,float16,fp8,0,4.016432126363118
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,64,128,1,float16,fp8,0,5.068304061889648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,8,128,1,float16,float16,0,8.90554682413737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,8,128,1,float16,fp8,0,8.36569086710612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,4,128,1,float16,fp8,0,8.559151967366537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,2,128,1,float16,float16,0,4.096469243367513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,2,128,1,float16,fp8,0,4.115914662679036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,4,128,1,float16,float16,0,4.134847958882649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,8,128,1,float16,float16,0,4.103290557861328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,4,128,1,float16,fp8,0,4.263973236083984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,64,128,1,float16,float16,0,2.3648160298665366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,64,128,1,float16,fp8,0,2.421610673268636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,1,128,1,float16,float16,0,2.1014453570048013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,8,128,1,float16,fp8,0,4.4254452387491865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,1,128,1,float16,fp8,0,2.054896036783854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,2,128,1,float16,float16,0,2.03276793162028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,64,128,1,float16,float16,0,1.2146026293436687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,64,128,1,float16,fp8,0,1.2643146514892578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,4,128,1,float16,float16,0,2.0560107231140137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,2,128,1,float16,fp8,0,2.1572373708089194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,8,128,1,float16,float16,0,2.02622397740682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,8,128,1,float16,fp8,0,2.2310187021891275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,4,128,1,float16,fp8,0,2.1817866961161294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,1,128,1,float16,float16,0,1.0040000279744465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,2,128,1,float16,float16,0,1.0147626399993896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,1,128,1,float16,fp8,0,1.0295573075612385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,2,128,1,float16,fp8,0,1.0518453121185303
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,4,128,1,float16,float16,0,1.0747093359629314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,8,128,1,float16,float16,0,1.0329759915669758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,4,128,1,float16,fp8,0,1.1126506328582764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,64,128,1,float16,fp8,0,0.649120012919108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,64,128,1,float16,float16,0,0.6267999807993571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,8,128,1,float16,fp8,0,1.1101813316345215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,1,128,1,float16,float16,0,0.5156586567560831
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,4,128,1,float16,float16,0,0.54585067431132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,1,128,1,float16,fp8,0,0.5488426685333252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,2,128,1,float16,float16,0,0.5339946746826172
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,2,128,1,float16,fp8,0,0.5540479818979899
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,8,128,1,float16,float16,0,0.5266773303349813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,4,128,1,float16,fp8,0,0.5679786602656046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,8,128,1,float16,fp8,0,0.5794026851654053
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,2,128,1,float16,float16,0,10.362933476765951
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,4,128,1,float16,fp8,0,10.198410669962565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,1,128,1,float16,float16,0,9.894778569539389
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,1,128,1,float16,fp8,0,9.434832255045572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,8,128,1,float16,fp8,0,10.056954701741537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,2,128,1,float16,fp8,0,9.713509241739908
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,4,128,1,float16,float16,0,10.430938720703125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,8,128,1,float16,float16,0,10.412426630655924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,64,128,1,float16,float16,0,5.923722585042317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,1,128,1,float16,float16,0,4.801994641621907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,2,128,1,float16,float16,0,4.820634524027507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,64,128,1,float16,fp8,0,5.920629501342773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,1,128,1,float16,fp8,0,4.704208056131999
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,2,128,1,float16,fp8,0,4.789877255757649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,4,128,1,float16,float16,0,5.021178563435872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,4,128,1,float16,fp8,0,5.096586545308431
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,1,128,1,float16,float16,0,2.30348269144694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,64,128,1,float16,float16,0,2.9496374130249023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,64,128,1,float16,fp8,0,2.986288070678711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,1,128,1,float16,fp8,0,2.377280076344808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,8,128,1,float16,float16,0,4.960010528564453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,2,128,1,float16,float16,0,2.407584031422933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,8,128,1,float16,fp8,0,5.021178563435872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,2,128,1,float16,fp8,0,2.417893409729004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,4,128,1,float16,float16,0,2.501743952433268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,64,128,1,float16,float16,0,1.5093386967976887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,8,128,1,float16,float16,0,2.37118927637736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,1,128,1,float16,float16,0,1.1658079624176025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,4,128,1,float16,fp8,0,2.6809441248575845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,64,128,1,float16,fp8,0,1.6322132746378581
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,1,128,1,float16,fp8,0,1.2152907053629558
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,8,128,1,float16,fp8,0,2.5198933283487954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,2,128,1,float16,float16,0,1.2517653306325276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,4,128,1,float16,float16,0,1.2498613198598225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,4,128,1,float16,fp8,0,1.3053653240203857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,2,128,1,float16,fp8,0,1.3388746579488118
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,8,128,1,float16,float16,0,1.2147520383199055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,64,128,1,float16,float16,0,0.7538666725158691
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,8,128,1,float16,fp8,0,1.3304853439331055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,1,128,1,float16,fp8,0,0.6416373252868652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,1,128,1,float16,float16,0,0.5898400147755941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,64,128,1,float16,fp8,0,0.8098506927490234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,2,128,1,float16,float16,0,0.6378399928410848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,2,128,1,float16,fp8,0,0.6498879988988241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,4,128,1,float16,float16,0,0.6353919903437296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,4,128,1,float16,fp8,0,0.6950399875640869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,8,128,1,float16,float16,0,0.6287946701049805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,8,128,1,float16,fp8,0,0.6675679683685303
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,64,128,1,float16,float16,0,0.4006933371225993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,64,128,1,float16,fp8,0,0.42803200085957843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,1,128,1,float16,float16,0,0.3136426607767741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,1,128,1,float16,fp8,0,0.33427198727925617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,4,128,1,float16,float16,0,0.32569066683451336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,2,128,1,float16,float16,0,0.31997867425282794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,2,128,1,float16,fp8,0,0.34699734052022296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,4,128,1,float16,fp8,0,0.36508798599243164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,8,128,1,float16,float16,0,0.32651732365290326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,8,128,1,float16,fp8,0,0.36500799655914307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,2,128,1,float16,float16,0,10.100010553995768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,2,128,1,float16,fp8,0,9.636650721232096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,4,128,1,float16,float16,0,10.307269414265951
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,8,128,1,float16,float16,0,9.926138559977213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,4,128,1,float16,fp8,0,10.307594935099283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,1,128,1,float16,fp8,0,9.36465581258138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,1,128,1,float16,float16,0,9.876768112182617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,1,128,1,float16,float16,0,4.629983901977539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,1,128,1,float16,fp8,0,4.859482765197754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,2,128,1,float16,float16,0,4.64464537302653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,64,128,1,float16,float16,0,6.093530654907227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,64,128,1,float16,fp8,0,6.285066604614258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,2,128,1,float16,fp8,0,4.765770594278972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,8,128,1,float16,fp8,0,10.047589619954428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,4,128,1,float16,float16,0,4.900863965352376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,4,128,1,float16,fp8,0,5.187365214029948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,64,128,1,float16,float16,0,3.0574986139933267
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,64,128,1,float16,fp8,0,3.163898785909017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,8,128,1,float16,float16,0,4.71342404683431
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,8,128,1,float16,fp8,0,5.256730715433757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,1,128,1,float16,float16,0,2.2022132873535156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,2,128,1,float16,float16,0,2.3363946278889975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,1,128,1,float16,fp8,0,2.3550240198771157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,2,128,1,float16,fp8,0,2.406053384145101
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,64,128,1,float16,float16,0,1.5465866724650066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,8,128,1,float16,float16,0,2.3946399688720703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,4,128,1,float16,float16,0,2.459946632385254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,8,128,1,float16,fp8,0,2.5839200019836426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,4,128,1,float16,fp8,0,2.5985120137532554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,1,128,1,float16,float16,0,1.1396479606628418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,1,128,1,float16,fp8,0,1.2001919746398926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,64,128,1,float16,fp8,0,1.6805599530537922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,2,128,1,float16,float16,0,1.1692426999409993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,4,128,1,float16,fp8,0,1.320847988128662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,4,128,1,float16,float16,0,1.2406453291575115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,2,128,1,float16,fp8,0,1.3097493648529053
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,8,128,1,float16,float16,0,1.209930658340454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,64,128,1,float16,float16,0,0.7825973033905029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,8,128,1,float16,fp8,0,1.3182186285654705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,1,128,1,float16,fp8,0,0.6178346474965414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,64,128,1,float16,fp8,0,0.8433653513590494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,1,128,1,float16,float16,0,0.5779893398284912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,2,128,1,float16,float16,0,0.6001120011011759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,2,128,1,float16,fp8,0,0.6581973234812418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,4,128,1,float16,float16,0,0.6247093280156454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,4,128,1,float16,fp8,0,0.6918719609578451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,8,128,1,float16,float16,0,0.6238933404286703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,8,128,1,float16,fp8,0,0.6678079764048258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,64,128,1,float16,float16,0,0.41105600198109943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,64,128,1,float16,fp8,0,0.44183464845021564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,1,128,1,float16,float16,0,0.30375466744105023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,1,128,1,float16,fp8,0,0.3266826669375102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,2,128,1,float16,float16,0,0.30645867188771564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,2,128,1,float16,fp8,0,0.3397866487503052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,4,128,1,float16,float16,0,0.3321066697438558
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,4,128,1,float16,fp8,0,0.3553546667098999
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,8,128,1,float16,float16,0,0.3248533407847087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,1,128,1,float16,fp8,0,0.18226132790247598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,8,128,1,float16,fp8,0,0.35869868596394855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,64,128,1,float16,float16,0,0.22038400173187256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,1,128,1,float16,float16,0,0.1646346648534139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,64,128,1,float16,fp8,0,0.24130666255950928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,2,128,1,float16,float16,0,0.17058134078979492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,2,128,1,float16,fp8,0,0.18201599518458048
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,4,128,1,float16,float16,0,0.18105065822601318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,4,128,1,float16,fp8,0,0.19891732931137085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,8,128,1,float16,float16,0,0.17522132396697998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,8,128,1,float16,fp8,0,0.1953493356704712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,1,128,1,float16,float16,0,5.5278879801432295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,1,128,1,float16,fp8,0,5.8454240163167315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,2,128,1,float16,float16,0,6.170207977294922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,4,128,1,float16,float16,0,6.262783686319987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,8,128,1,float16,float16,0,5.8343251546223955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,2,128,1,float16,fp8,0,5.998661041259766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,4,128,1,float16,fp8,0,6.61903444925944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,8,128,1,float16,fp8,0,6.402661641438802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,64,128,1,float16,float16,0,4.145888010660808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,1,128,1,float16,float16,0,2.8500000635782876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,2,128,1,float16,float16,0,2.910005251566569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,64,128,1,float16,fp8,0,4.176346778869629
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,1,128,1,float16,fp8,0,3.0421441396077475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,4,128,1,float16,float16,0,3.030917485555013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,2,128,1,float16,fp8,0,3.000816027323405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,4,128,1,float16,fp8,0,3.3047253290812173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,1,128,1,float16,float16,0,1.3912639617919922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,64,128,1,float16,float16,0,1.979248046875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,8,128,1,float16,float16,0,3.0141067504882812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,1,128,1,float16,fp8,0,1.4835467338562012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,64,128,1,float16,fp8,0,2.106943925221761
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,8,128,1,float16,fp8,0,3.2904265721639
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,2,128,1,float16,fp8,0,1.548085371653239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,2,128,1,float16,float16,0,1.4289226531982422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,4,128,1,float16,float16,0,1.5098986625671387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,64,128,1,float16,float16,0,0.9955039819081625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,8,128,1,float16,float16,0,1.484447956085205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,4,128,1,float16,fp8,0,1.836634635925293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,8,128,1,float16,fp8,0,1.6177280743916829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,64,128,1,float16,fp8,0,1.0891413688659668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,1,128,1,float16,float16,0,0.6970773537953695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,2,128,1,float16,float16,0,0.7272000312805176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,1,128,1,float16,fp8,0,0.7538719971974691
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,4,128,1,float16,float16,0,0.7899573644002279
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,2,128,1,float16,fp8,0,0.7838133176167806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,8,128,1,float16,float16,0,0.7566293080647787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,4,128,1,float16,fp8,0,0.8493653138478597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,8,128,1,float16,fp8,0,0.831285317738851
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,64,128,1,float16,float16,0,0.5157279968261719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,1,128,1,float16,fp8,0,0.3992053270339966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,2,128,1,float16,float16,0,0.3749013344446818
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,1,128,1,float16,float16,0,0.36988266309102374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,64,128,1,float16,fp8,0,0.5564959843953451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,2,128,1,float16,fp8,0,0.4032426675160726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,4,128,1,float16,float16,0,0.39773333072662354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,4,128,1,float16,fp8,0,0.4442346493403117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,64,128,1,float16,float16,0,0.2710826595624288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,8,128,1,float16,fp8,0,0.43800000349680585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,8,128,1,float16,float16,0,0.3941333293914795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,64,128,1,float16,fp8,0,0.29999999205271405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,1,128,1,float16,float16,0,0.18830400705337524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,1,128,1,float16,fp8,0,0.21330666542053223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,2,128,1,float16,float16,0,0.1981226603190104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,2,128,1,float16,fp8,0,0.22032000621159872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,4,128,1,float16,float16,0,0.2146773338317871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,4,128,1,float16,fp8,0,0.23617066939671835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,8,128,1,float16,float16,0,0.21214399735132852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,8,128,1,float16,fp8,0,0.2358186642328898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,2,128,1,float16,float16,0,0.110944002866745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,1,128,1,float16,float16,0,0.10530666510264079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,64,128,1,float16,float16,0,0.15003732840220133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,64,128,1,float16,fp8,0,0.1641546686490377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,1,128,1,float16,fp8,0,0.11925333738327026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,2,128,1,float16,fp8,0,0.1232266624768575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,4,128,1,float16,float16,0,0.11947733163833618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,8,128,1,float16,float16,0,0.1169546643892924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,4,128,1,float16,fp8,0,0.13210666179656982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,8,128,1,float16,fp8,0,0.12710932890574136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,1,128,1,float16,fp8,0,6.229167938232422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,1,128,1,float16,float16,0,5.889253616333008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,2,128,1,float16,fp8,0,6.411386489868164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,8,128,1,float16,fp8,0,6.8526611328125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,2,128,1,float16,float16,0,6.0987199147542315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,8,128,1,float16,float16,0,6.585877100626628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,4,128,1,float16,fp8,0,7.303461074829102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,64,128,1,float16,float16,0,4.394517262776692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,1,128,1,float16,float16,0,2.807530721028646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,1,128,1,float16,fp8,0,3.1227893829345703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,2,128,1,float16,float16,0,2.936784108479818
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,2,128,1,float16,fp8,0,3.318042755126953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,64,128,1,float16,fp8,0,4.863775889078776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,4,128,1,float16,float16,0,3.2126026153564453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,4,128,1,float16,float16,0,6.485706965128581
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,1,128,1,float16,float16,0,1.4081546465555828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,8,128,1,float16,float16,0,3.1009066899617515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,8,128,1,float16,fp8,0,3.393312136332194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,64,128,1,float16,float16,0,2.1957546869913735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,1,128,1,float16,fp8,0,1.5938026110331218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,4,128,1,float16,fp8,0,3.7035306294759116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,64,128,1,float16,fp8,0,2.4784480730692544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,2,128,1,float16,fp8,0,1.6156214078267415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,2,128,1,float16,float16,0,1.5892747243245442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,4,128,1,float16,float16,0,1.634640057881673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,8,128,1,float16,float16,0,1.5454400380452473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,4,128,1,float16,fp8,0,1.803269386291504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,8,128,1,float16,fp8,0,1.7687360445658367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,1,128,1,float16,float16,0,0.7161920070648193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,64,128,1,float16,fp8,0,1.2389919757843018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,64,128,1,float16,float16,0,1.1346773306528728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,1,128,1,float16,fp8,0,0.8032320340474447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,2,128,1,float16,float16,0,0.7681333223978678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,8,128,1,float16,float16,0,0.7854453722635905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,4,128,1,float16,float16,0,0.8175413608551025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,4,128,1,float16,fp8,0,0.9333439668019613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,2,128,1,float16,fp8,0,0.8292586803436279
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,1,128,1,float16,float16,0,0.3786453406016032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,64,128,1,float16,fp8,0,0.6374933322270712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,2,128,1,float16,float16,0,0.3884853521982829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,64,128,1,float16,float16,0,0.5754186709721884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,8,128,1,float16,fp8,0,0.894922653834025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,1,128,1,float16,fp8,0,0.41260798772176105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,2,128,1,float16,fp8,0,0.4289919932683309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,4,128,1,float16,float16,0,0.4248373508453369
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,8,128,1,float16,float16,0,0.40855999787648517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,8,128,1,float16,fp8,0,0.4649920066197713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,4,128,1,float16,fp8,0,0.4748266537984212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,64,128,1,float16,fp8,0,0.3319786588350932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,64,128,1,float16,float16,0,0.3014133373896281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,1,128,1,float16,float16,0,0.19875733057657877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,2,128,1,float16,float16,0,0.20995734135309854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,1,128,1,float16,fp8,0,0.22097599506378174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,4,128,1,float16,float16,0,0.2239733338356018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,2,128,1,float16,fp8,0,0.23418132464090982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,4,128,1,float16,fp8,0,0.25645333528518677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,8,128,1,float16,float16,0,0.2201813260714213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,8,128,1,float16,fp8,0,0.24661866823832193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,64,128,1,float16,float16,0,0.1646453340848287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,64,128,1,float16,fp8,0,0.18332799275716147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,1,128,1,float16,float16,0,0.10820800065994263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,1,128,1,float16,fp8,0,0.12813333670298258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,2,128,1,float16,float16,0,0.11409599582354228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,2,128,1,float16,fp8,0,0.12826666235923767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,4,128,1,float16,float16,0,0.12442133824030559
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,4,128,1,float16,fp8,0,0.13980266451835632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,8,128,1,float16,fp8,0,0.1365173359711965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,8,128,1,float16,float16,0,0.12107200423876445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,64,128,1,float16,fp8,0,0.10725333293279012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,64,128,1,float16,float16,0,0.09222933650016785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,1,128,1,float16,fp8,0,0.06610666712125142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,1,128,1,float16,float16,0,0.059802666306495667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,2,128,1,float16,float16,0,0.06498666604359944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,2,128,1,float16,fp8,0,0.0717493345340093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,4,128,1,float16,float16,0,0.07131200035413106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,4,128,1,float16,fp8,0,0.07819200058778127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,8,128,1,float16,float16,0,0.06638399759928386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,8,128,1,float16,fp8,0,0.0751146674156189
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,2,128,1,float16,float16,0,4.432778676350911
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,2,128,1,float16,fp8,0,4.950069427490234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,4,128,1,float16,float16,0,4.9811147054036455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,8,128,1,float16,float16,0,4.654565175374349
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,1,128,1,float16,float16,0,2.096165339152018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,64,128,1,float16,float16,0,3.6252479553222656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,1,128,1,float16,fp8,0,4.7072906494140625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,64,128,1,float16,fp8,0,4.31437333424886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,1,128,1,float16,float16,0,4.224229176839192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,4,128,1,float16,fp8,0,5.733413060506185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,2,128,1,float16,float16,0,2.2171680132548013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,2,128,1,float16,fp8,0,2.4459840456644693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,4,128,1,float16,float16,0,2.4689173698425293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,8,128,1,float16,fp8,0,2.667781194051107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,64,128,1,float16,float16,0,1.821466604868571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,4,128,1,float16,fp8,0,2.8800427118937173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,1,128,1,float16,float16,0,1.0556800365447998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,1,128,1,float16,fp8,0,2.370240052541097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,8,128,1,float16,float16,0,2.3079466819763184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,64,128,1,float16,fp8,0,2.1748533248901367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,2,128,1,float16,float16,0,1.1172746817270915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,2,128,1,float16,fp8,0,1.238213300704956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,4,128,1,float16,float16,0,1.2462986310323079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,1,128,1,float16,fp8,0,1.1940266291300456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,8,128,1,float16,float16,0,1.1827786763509114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,4,128,1,float16,fp8,0,1.4448107083638508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,64,128,1,float16,float16,0,0.9386613368988037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,1,128,1,float16,fp8,0,0.6095360120137533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,1,128,1,float16,float16,0,0.5401440064112345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,64,128,1,float16,fp8,0,1.099503993988037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,8,128,1,float16,fp8,0,5.346373240152995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,2,128,1,float16,float16,0,0.5711679855982462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,2,128,1,float16,fp8,0,0.6321973403294882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,4,128,1,float16,fp8,0,0.7333013216654459
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,8,128,1,float16,float16,0,0.6014879941940308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,1,128,1,float16,float16,0,0.28090665737787884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,4,128,1,float16,float16,0,0.635312000910441
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,64,128,1,float16,fp8,0,0.5615520079930624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,1,128,1,float16,fp8,0,0.3179146647453308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,8,128,1,float16,fp8,0,1.3913119633992512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,8,128,1,float16,fp8,0,0.7019786834716797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,2,128,1,float16,fp8,0,0.32897599538167316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,2,128,1,float16,float16,0,0.29621867338816327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,4,128,1,float16,float16,0,0.32850666840871173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,8,128,1,float16,float16,0,0.3109760085741679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,4,128,1,float16,fp8,0,0.37969064712524414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,64,128,1,float16,fp8,0,0.2945760091145833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,1,128,1,float16,float16,0,0.14939733346303305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,1,128,1,float16,fp8,0,0.17459734280904135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,64,128,1,float16,float16,0,0.2507413427035014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,8,128,1,float16,fp8,0,0.36667199929555255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,2,128,1,float16,float16,0,0.15785066286722818
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,2,128,1,float16,fp8,0,0.17671465873718262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,4,128,1,float16,fp8,0,0.20017067591349283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,4,128,1,float16,float16,0,0.1742186745007833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,64,128,1,float16,float16,0,0.4728906551996867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,8,128,1,float16,float16,0,0.168778657913208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,1,128,1,float16,fp8,0,0.09646933277448018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,64,128,1,float16,float16,0,0.13332266608874002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,64,128,1,float16,fp8,0,0.16087999939918518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,1,128,1,float16,float16,0,0.0853706697622935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,8,128,1,float16,fp8,0,0.19423999389012656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,2,128,1,float16,fp8,0,0.10029333829879761
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,2,128,1,float16,float16,0,0.0883626639842987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,4,128,1,float16,fp8,0,0.11008532842000325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,4,128,1,float16,float16,0,0.09661333759625752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,8,128,1,float16,float16,0,0.09124267101287842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,8,128,1,float16,fp8,0,0.10732799768447876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,64,128,1,float16,float16,0,0.07664533456166585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,64,128,1,float16,fp8,0,0.09422399600346883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,1,128,1,float16,float16,0,0.04674133161703745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,1,128,1,float16,fp8,0,0.05286933481693268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,2,128,1,float16,float16,0,0.05007466673851013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,2,128,1,float16,fp8,0,0.05649599929650625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,4,128,1,float16,float16,0,0.053957333167394005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,4,128,1,float16,fp8,0,0.060032000144322716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,8,128,1,float16,fp8,0,0.05699199934800466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,8,128,1,float16,float16,0,0.05167999863624573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,64,128,1,float16,float16,0,0.04460800190766653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,64,128,1,float16,fp8,0,0.05308799942334493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,1,128,1,float16,float16,0,0.027552001178264618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,1,128,1,float16,fp8,0,0.03323200096686681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,2,128,1,float16,float16,0,0.030218665798505146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,2,128,1,float16,fp8,0,0.03435733417669932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,4,128,1,float16,float16,0,0.03183999905983607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,4,128,1,float16,fp8,0,0.03618133316437403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,8,128,1,float16,float16,0,0.03181333343187968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,8,128,1,float16,fp8,0,0.035877334574858345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,1,128,1,float16,float16,0,1.801743984222412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,2,128,1,float16,float16,0,1.8665013313293457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,1,128,1,float16,fp8,0,2.025397300720215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,4,128,1,float16,fp8,0,2.5153493881225586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,2,128,1,float16,fp8,0,2.0482346216837564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,64,128,1,float16,float16,0,1.6739039421081543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,64,128,1,float16,fp8,0,1.8990987141927083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,1,128,1,float16,fp8,0,1.0139679908752441
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,8,128,1,float16,float16,0,1.9603519439697266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,2,128,1,float16,float16,0,0.9453653494517008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,8,128,1,float16,fp8,0,2.291850725809733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,1,128,1,float16,float16,0,0.9151573181152344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,4,128,1,float16,float16,0,2.126848061879476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,2,128,1,float16,fp8,0,1.0403892993927002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,4,128,1,float16,fp8,0,1.2564319769541423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,8,128,1,float16,float16,0,1.0085386435190837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,64,128,1,float16,fp8,0,0.9593546390533447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,64,128,1,float16,float16,0,0.8470719655354818
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,8,128,1,float16,fp8,0,1.1921013196309407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,4,128,1,float16,float16,0,1.0782933235168457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,2,128,1,float16,float16,0,0.4807200034459432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,1,128,1,float16,fp8,0,0.5204319953918457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,1,128,1,float16,float16,0,0.4633546670277913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,2,128,1,float16,fp8,0,0.5310613314310709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,8,128,1,float16,float16,0,0.5147733290990194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,4,128,1,float16,fp8,0,0.6380426486333212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,64,128,1,float16,float16,0,0.433296004931132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,4,128,1,float16,float16,0,0.5491626660029093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,8,128,1,float16,fp8,0,0.606053352355957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,1,128,1,float16,float16,0,0.24127467473347983
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,1,128,1,float16,fp8,0,0.2714186708132426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,64,128,1,float16,fp8,0,0.49038398265838623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,2,128,1,float16,float16,0,0.2507893244425456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,4,128,1,float16,float16,0,0.28295467297236127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,2,128,1,float16,fp8,0,0.2797866662343343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,64,128,1,float16,float16,0,0.22696000337600708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,64,128,1,float16,fp8,0,0.2596213420232137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,4,128,1,float16,fp8,0,0.3320373296737671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,8,128,1,float16,float16,0,0.2690986593564351
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,8,128,1,float16,fp8,0,0.31706666946411133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,1,128,1,float16,float16,0,0.13034666577974954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,1,128,1,float16,fp8,0,0.14779733618100485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,2,128,1,float16,float16,0,0.13522666692733765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,2,128,1,float16,fp8,0,0.15238933761914572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,4,128,1,float16,fp8,0,0.17706133921941122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,8,128,1,float16,float16,0,0.1423466702302297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,4,128,1,float16,float16,0,0.15031466881434122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,8,128,1,float16,fp8,0,0.16863999764124551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,64,128,1,float16,float16,0,0.12358400225639343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,64,128,1,float16,fp8,0,0.1421119968096415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,1,128,1,float16,float16,0,0.0738613357146581
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,1,128,1,float16,fp8,0,0.08569066723187764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,2,128,1,float16,float16,0,0.07497600217660268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,2,128,1,float16,fp8,0,0.08756800492604573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,4,128,1,float16,float16,0,0.08367466926574707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,4,128,1,float16,fp8,0,0.09827199578285217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,8,128,1,float16,float16,0,0.07884799937407176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,64,128,1,float16,float16,0,0.07197333375612895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,8,128,1,float16,fp8,0,0.09443199634552002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,64,128,1,float16,fp8,0,0.08330133557319641
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,1,128,1,float16,float16,0,0.039850667119026184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,1,128,1,float16,fp8,0,0.0460746685663859
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,2,128,1,float16,float16,0,0.04135466615358988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,2,128,1,float16,fp8,0,0.04750399788220724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,4,128,1,float16,float16,0,0.0469706654548645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,4,128,1,float16,fp8,0,0.05425600210825602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,8,128,1,float16,float16,0,0.04110399881998698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,8,128,1,float16,fp8,0,0.048895999789237976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,64,128,1,float16,float16,0,0.03946666667858759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,64,128,1,float16,fp8,0,0.04433066646258036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,1,128,1,float16,float16,0,0.024538666009902954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,1,128,1,float16,fp8,0,0.02888533224662145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,2,128,1,float16,float16,0,0.024693332612514496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,2,128,1,float16,fp8,0,0.02937600016593933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,4,128,1,float16,float16,0,0.025466665625572205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,4,128,1,float16,fp8,0,0.03028800090154012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,8,128,1,float16,fp8,0,0.030181333422660828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,8,128,1,float16,float16,0,0.025562666356563568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,64,128,1,float16,float16,0,0.019685332973798115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,64,128,1,float16,fp8,0,0.024821333587169647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,1,128,1,float16,float16,0,0.017029333859682083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,1,128,1,float16,fp8,0,0.019744000087181728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,2,128,1,float16,float16,0,0.017055999487638474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,8,128,1,float16,float16,0,0.01836266616980235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,2,128,1,float16,fp8,0,0.01998399943113327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,4,128,1,float16,float16,0,0.01738133281469345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,4,128,1,float16,fp8,0,0.02015999952952067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,8,128,1,float16,fp8,0,0.01987733319401741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,1,128,1,float16,float16,0,0.8361013730367025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,1,128,1,float16,fp8,0,0.8859253724416097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,2,128,1,float16,float16,0,0.8970613479614258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,2,128,1,float16,fp8,0,0.9326240221659342
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,4,128,1,float16,float16,0,1.0370399951934814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,4,128,1,float16,fp8,0,1.1475733121236165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,64,128,1,float16,float16,0,0.8103306293487549
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,8,128,1,float16,fp8,0,1.037061293919881
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,1,128,1,float16,float16,0,0.42639466126759845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,64,128,1,float16,fp8,0,0.8528532981872559
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,1,128,1,float16,fp8,0,0.46113065878550213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,2,128,1,float16,float16,0,0.45661334196726483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,2,128,1,float16,fp8,0,0.4841066598892212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,4,128,1,float16,fp8,0,0.5862186749776205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,64,128,1,float16,float16,0,0.4149386485417684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,4,128,1,float16,float16,0,0.5250879923502604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,8,128,1,float16,float16,0,1.0021973450978596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,8,128,1,float16,fp8,0,0.5252213478088379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,1,128,1,float16,float16,0,0.22116265694300333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,8,128,1,float16,float16,0,0.5131040016810099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,64,128,1,float16,fp8,0,0.44649068514506024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,2,128,1,float16,float16,0,0.2396213412284851
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,1,128,1,float16,fp8,0,0.24594134092330933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,4,128,1,float16,fp8,0,0.3029759923617045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,4,128,1,float16,float16,0,0.2720800042152405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,2,128,1,float16,fp8,0,0.2579413255055745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,8,128,1,float16,float16,0,0.266213337580363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,64,128,1,float16,float16,0,0.21864000956217447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,64,128,1,float16,fp8,0,0.2363626758257548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,1,128,1,float16,float16,0,0.11877333124478658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,1,128,1,float16,fp8,0,0.13499733805656433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,2,128,1,float16,float16,0,0.12869866689046225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,2,128,1,float16,fp8,0,0.14309333761533102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,8,128,1,float16,fp8,0,0.28044267495473224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,4,128,1,float16,float16,0,0.1450933317343394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,4,128,1,float16,fp8,0,0.16596266627311707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,8,128,1,float16,fp8,0,0.14963199694951376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,1,128,1,float16,fp8,0,0.07878399888674419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,2,128,1,float16,float16,0,0.07257066667079926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,64,128,1,float16,float16,0,0.11891733606656392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,64,128,1,float16,fp8,0,0.12785067160924277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,8,128,1,float16,float16,0,0.13969600200653076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,4,128,1,float16,fp8,0,0.0932426651318868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,8,128,1,float16,float16,0,0.07396266857783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,1,128,1,float16,float16,0,0.06922133266925812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,2,128,1,float16,fp8,0,0.08210133512814839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,4,128,1,float16,float16,0,0.07972266773382823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,8,128,1,float16,fp8,0,0.08490133285522461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,64,128,1,float16,float16,0,0.06774933139483134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,1,128,1,float16,float16,0,0.03469866762558619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,64,128,1,float16,fp8,0,0.07330133517583211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,1,128,1,float16,fp8,0,0.03872533390919367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,2,128,1,float16,float16,0,0.03756266583998998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,2,128,1,float16,fp8,0,0.04075733323891958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,4,128,1,float16,float16,0,0.041989331444104515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,4,128,1,float16,fp8,0,0.045978665351867676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,8,128,1,float16,float16,0,0.03741333385308584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,64,128,1,float16,float16,0,0.03846399982770284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,8,128,1,float16,fp8,0,0.04331733286380768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,64,128,1,float16,fp8,0,0.039749334255854286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,1,128,1,float16,float16,0,0.021386665602525074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,1,128,1,float16,fp8,0,0.02497600018978119
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,2,128,1,float16,float16,0,0.021525333325068157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,2,128,1,float16,fp8,0,0.024735999604066212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,4,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,4,128,1,float16,fp8,0,0.026352000733216602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,8,128,1,float16,float16,0,0.023141334454218548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,8,128,1,float16,fp8,0,0.02611200014750163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,64,128,1,float16,float16,0,0.018618666877349217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,2,128,1,float16,fp8,0,0.018415999909241993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,64,128,1,float16,fp8,0,0.020629333953062694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,1,128,1,float16,float16,0,0.015605332950750986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,1,128,1,float16,fp8,0,0.01828266680240631
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,2,128,1,float16,float16,0,0.01573333392540614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,4,128,1,float16,float16,0,0.01599466676513354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,4,128,1,float16,fp8,0,0.018613333503405254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,8,128,1,float16,float16,0,0.015840000162522
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,8,128,1,float16,fp8,0,0.01811733345190684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,64,128,1,float16,float16,0,0.016751999656359356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,64,128,1,float16,fp8,0,0.019359999646743137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,1,128,1,float16,float16,0,0.01516266663869222
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,1,128,1,float16,fp8,0,0.017914666483799618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,2,128,1,float16,float16,0,0.015392000476519266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,2,128,1,float16,fp8,0,0.01826133330663045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,4,128,1,float16,float16,0,0.015333333363135656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,4,128,1,float16,fp8,0,0.017973333597183228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,8,128,1,float16,float16,0,0.015279999623696009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,8,128,1,float16,fp8,0,0.01730666682124138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,1,128,1,float16,float16,0,0.426421324412028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,2,128,1,float16,float16,0,0.4598879814147949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,2,128,1,float16,fp8,0,0.48519468307495117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,8,128,1,float16,fp8,0,0.526367982228597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,1,128,1,float16,fp8,0,0.45680534839630127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,64,128,1,float16,float16,0,0.46464534600575763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,4,128,1,float16,fp8,0,0.5801440080006918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,8,128,1,float16,float16,0,0.5042453209559122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,4,128,1,float16,float16,0,0.522490660349528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,64,128,1,float16,fp8,0,0.4440906842549642
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,1,128,1,float16,float16,0,0.22164799769719443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,1,128,1,float16,fp8,0,0.24441067377726236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,64,128,1,float16,float16,0,0.24181334177652994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,2,128,1,float16,fp8,0,0.2565386692682902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,64,128,1,float16,fp8,0,0.22882666190465292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,4,128,1,float16,fp8,0,0.3073226610819499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,4,128,1,float16,float16,0,0.27164266506830853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,8,128,1,float16,float16,0,0.25942399104436237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,2,128,1,float16,float16,0,0.23829867442448935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,1,128,1,float16,fp8,0,0.13737600048383078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,2,128,1,float16,fp8,0,0.14273599783579508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,4,128,1,float16,float16,0,0.144378662109375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,8,128,1,float16,float16,0,0.138154665629069
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,2,128,1,float16,float16,0,0.12799466649691263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,4,128,1,float16,fp8,0,0.16646933555603027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,1,128,1,float16,float16,0,0.12012267112731934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,8,128,1,float16,fp8,0,0.1508746643861135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,8,128,1,float16,fp8,0,0.27473066250483197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,64,128,1,float16,float16,0,0.12938666343688965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,64,128,1,float16,fp8,0,0.12231466174125671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,1,128,1,float16,float16,0,0.06759466727574666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,1,128,1,float16,fp8,0,0.07813866436481476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,2,128,1,float16,float16,0,0.07296533385912578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,2,128,1,float16,fp8,0,0.08147199948628743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,4,128,1,float16,float16,0,0.0799786647160848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,4,128,1,float16,fp8,0,0.09273599584897359
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,64,128,1,float16,fp8,0,0.06926933427651723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,1,128,1,float16,float16,0,0.03426666557788849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,64,128,1,float16,float16,0,0.07204799850781758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,2,128,1,float16,float16,0,0.03711466739575068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,8,128,1,float16,float16,0,0.072202667593956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,8,128,1,float16,fp8,0,0.0843946635723114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,1,128,1,float16,fp8,0,0.038704000413417816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,2,128,1,float16,fp8,0,0.040106666584809623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,8,128,1,float16,fp8,0,0.04233066737651825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,4,128,1,float16,float16,0,0.042250668009122215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,4,128,1,float16,fp8,0,0.04363200068473816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,8,128,1,float16,float16,0,0.03722133239110311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,64,128,1,float16,float16,0,0.04131733377774557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,64,128,1,float16,fp8,0,0.036576000352700554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,1,128,1,float16,float16,0,0.021231998999913532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,1,128,1,float16,fp8,0,0.02476266771554947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,2,128,1,float16,float16,0,0.02139200021823247
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,2,128,1,float16,fp8,0,0.02481599897146225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,4,128,1,float16,float16,0,0.02279466638962428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,4,128,1,float16,fp8,0,0.02643733223279317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,64,128,1,float16,float16,0,0.019248000035683315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,8,128,1,float16,float16,0,0.023178666830062866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,8,128,1,float16,fp8,0,0.026191999514897663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,2,128,1,float16,fp8,0,0.018405333161354065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,1,128,1,float16,float16,0,0.015498666713635126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,64,128,1,float16,fp8,0,0.02004266654451688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,1,128,1,float16,fp8,0,0.018496000518401463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,2,128,1,float16,float16,0,0.015770666301250458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,4,128,1,float16,fp8,0,0.018250666558742523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,4,128,1,float16,float16,0,0.015717333803574245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,8,128,1,float16,float16,0,0.01607999950647354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,8,128,1,float16,fp8,0,0.018112000077962875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,64,128,1,float16,float16,0,0.014736000448465347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,64,128,1,float16,fp8,0,0.015461333096027374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,1,128,1,float16,float16,0,0.015344000111023584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,1,128,1,float16,fp8,0,0.017914666483799618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,2,128,1,float16,float16,0,0.015562667200962702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,2,128,1,float16,fp8,0,0.017914666483799618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,4,128,1,float16,float16,0,0.015200000256299973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,4,128,1,float16,fp8,0,0.017621333400408428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,8,128,1,float16,float16,0,0.015477333217859268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,8,128,1,float16,fp8,0,0.01770666614174843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,64,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,64,128,1,float16,fp8,0,0.014906667172908783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,1,128,1,float16,fp8,0,0.01781333362062772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,1,128,1,float16,float16,0,0.015173333386580149
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,2,128,1,float16,float16,0,0.015402667224407196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,2,128,1,float16,fp8,0,0.01746133342385292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,4,128,1,float16,float16,0,0.015157333264748255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,4,128,1,float16,fp8,0,0.017370666066805523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,8,128,1,float16,float16,0,0.014869333555301031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,8,128,1,float16,fp8,0,0.017738666385412216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,2,128,1,float16,float16,0,0.23875200748443604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,2,128,1,float16,fp8,0,0.2560373346010844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,4,128,1,float16,float16,0,0.26844267050425213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,1,128,1,float16,float16,0,0.22084800402323404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,8,128,1,float16,float16,0,0.2594613234202067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,8,128,1,float16,fp8,0,0.27396267652511597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,4,128,1,float16,fp8,0,0.308896005153656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,64,128,1,float16,float16,0,0.3420586585998535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,1,128,1,float16,fp8,0,0.2467893362045288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,64,128,1,float16,fp8,0,0.3216266632080078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,1,128,1,float16,float16,0,0.11819733182589214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,1,128,1,float16,fp8,0,0.1368053356806437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,2,128,1,float16,float16,0,0.12800000111262003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,2,128,1,float16,fp8,0,0.14325867096583048
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,4,128,1,float16,float16,0,0.14407466848691305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,4,128,1,float16,fp8,0,0.16597867012023926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,64,128,1,float16,float16,0,0.1771413286526998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,8,128,1,float16,fp8,0,0.14865600069363913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,64,128,1,float16,fp8,0,0.1711840033531189
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,1,128,1,float16,float16,0,0.0689386675755183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,1,128,1,float16,fp8,0,0.07899733384450276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,8,128,1,float16,float16,0,0.13664533694585165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,2,128,1,float16,float16,0,0.07241066793600719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,4,128,1,float16,float16,0,0.08016533156236012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,2,128,1,float16,fp8,0,0.0820000022649765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,4,128,1,float16,fp8,0,0.09364799658457439
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,8,128,1,float16,float16,0,0.07202133536338806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,8,128,1,float16,fp8,0,0.08306133250395457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,64,128,1,float16,fp8,0,0.09343467156092326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,64,128,1,float16,float16,0,0.09578667084376018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,1,128,1,float16,float16,0,0.03463999927043915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,1,128,1,float16,fp8,0,0.03844266633192698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,2,128,1,float16,float16,0,0.0373279998699824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,2,128,1,float16,fp8,0,0.039621333281199135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,4,128,1,float16,float16,0,0.04197866717974345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,4,128,1,float16,fp8,0,0.04433066646258036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,8,128,1,float16,float16,0,0.037130666275819145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,8,128,1,float16,fp8,0,0.04214400053024292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,64,128,1,float16,fp8,0,0.04878933231035868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,64,128,1,float16,float16,0,0.0536106675863266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,1,128,1,float16,float16,0,0.021365332106749218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,1,128,1,float16,fp8,0,0.024874667326609295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,2,128,1,float16,float16,0,0.02144533395767212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,2,128,1,float16,fp8,0,0.024703999360402424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,4,128,1,float16,float16,0,0.022842665513356526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,4,128,1,float16,fp8,0,0.026416001220544178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,8,128,1,float16,float16,0,0.022821334501107533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,64,128,1,float16,float16,0,0.02664533257484436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,8,128,1,float16,fp8,0,0.02589866767326991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,1,128,1,float16,float16,0,0.015583999454975128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,64,128,1,float16,fp8,0,0.026346666117509205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,1,128,1,float16,fp8,0,0.01841066653529803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,2,128,1,float16,float16,0,0.015487999965747198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,2,128,1,float16,fp8,0,0.018186666071414948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,8,128,1,float16,float16,0,0.01569066693385442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,4,128,1,float16,float16,0,0.016143999993801117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,8,128,1,float16,fp8,0,0.018053332964579265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,4,128,1,float16,fp8,0,0.018309333672126133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,64,128,1,float16,float16,0,0.018031999468803406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,64,128,1,float16,fp8,0,0.01883200059334437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,4,128,1,float16,float16,0,0.015173333386580149
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,1,128,1,float16,float16,0,0.01534933348496755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,1,128,1,float16,fp8,0,0.017952000101407368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,2,128,1,float16,float16,0,0.015173333386580149
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,2,128,1,float16,fp8,0,0.017727999637524288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,4,128,1,float16,fp8,0,0.01806933308641116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,8,128,1,float16,fp8,0,0.017808000246683758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,8,128,1,float16,float16,0,0.01565333331624667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,64,128,1,float16,fp8,0,0.015040000279744467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,64,128,1,float16,float16,0,0.013936000565687815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,1,128,1,float16,fp8,0,0.017237332959969837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,1,128,1,float16,float16,0,0.014874666929244995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,2,128,1,float16,float16,0,0.015077333897352219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,2,128,1,float16,fp8,0,0.017301333447297413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,4,128,1,float16,float16,0,0.015114666273196539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,4,128,1,float16,fp8,0,0.017583999782800674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,8,128,1,float16,float16,0,0.014991999914248785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,8,128,1,float16,fp8,0,0.017130666722853977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,64,128,1,float16,float16,0,0.013786666095256805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,2,128,1,float16,fp8,0,0.016143999993801117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,64,128,1,float16,fp8,0,0.014581333845853806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,1,128,1,float16,float16,0,0.015381333728631338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,1,128,1,float16,fp8,0,0.016352000335852306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,2,128,1,float16,float16,0,0.014842666685581207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,4,128,1,float16,float16,0,0.014495999862750372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,8,128,1,float16,float16,0,0.01470400020480156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,4,128,1,float16,fp8,0,0.01634666696190834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,8,128,1,float16,fp8,0,0.016149333367745083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,2,128,1,float16,fp8,0,0.143994669119517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,4,128,1,float16,float16,0,0.14400532841682434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,1,128,1,float16,fp8,0,0.13729600111643472
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,4,128,1,float16,fp8,0,0.16660799582799277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,1,128,1,float16,float16,0,0.11820800105730693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,2,128,1,float16,float16,0,0.12762133280436197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,8,128,1,float16,float16,0,0.1336426635583242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,64,128,1,float16,float16,0,0.2791253328323364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,64,128,1,float16,fp8,0,0.2689173420270284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,1,128,1,float16,float16,0,0.06926933427651723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,1,128,1,float16,fp8,0,0.07854400078455608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,8,128,1,float16,fp8,0,0.14845333496729532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,2,128,1,float16,float16,0,0.0721973329782486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,8,128,1,float16,fp8,0,0.08323733508586884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,2,128,1,float16,fp8,0,0.08203733464082082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,64,128,1,float16,float16,0,0.14577066898345947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,4,128,1,float16,float16,0,0.08027733365694682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,1,128,1,float16,float16,0,0.03404266635576884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,8,128,1,float16,float16,0,0.07177599767843883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,2,128,1,float16,float16,0,0.03688533355792364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,64,128,1,float16,fp8,0,0.14102400342623392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,4,128,1,float16,fp8,0,0.09353599945704143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,1,128,1,float16,fp8,0,0.03915733347336451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,4,128,1,float16,float16,0,0.04214400053024292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,2,128,1,float16,fp8,0,0.040778666734695435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,8,128,1,float16,fp8,0,0.04197866717974345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,4,128,1,float16,fp8,0,0.04412800073623657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,8,128,1,float16,float16,0,0.03643733263015747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,64,128,1,float16,float16,0,0.07860266665617625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,64,128,1,float16,fp8,0,0.0735999991496404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,1,128,1,float16,float16,0,0.021173333128293354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,2,128,1,float16,float16,0,0.021407999098300934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,1,128,1,float16,fp8,0,0.02475200096766154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,2,128,1,float16,fp8,0,0.02455466737349828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,4,128,1,float16,float16,0,0.022677332162857056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,4,128,1,float16,fp8,0,0.026672000686327618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,8,128,1,float16,float16,0,0.023024000227451324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,64,128,1,float16,fp8,0,0.03807466725508372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,8,128,1,float16,fp8,0,0.025754667818546295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,64,128,1,float16,float16,0,0.03806933263937632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,1,128,1,float16,float16,0,0.015541333705186844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,1,128,1,float16,fp8,0,0.01830400029818217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,4,128,1,float16,fp8,0,0.018245333184798557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,2,128,1,float16,float16,0,0.015749332805474598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,2,128,1,float16,fp8,0,0.0183146670460701
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,4,128,1,float16,float16,0,0.01597333326935768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,8,128,1,float16,float16,0,0.015850666910409927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,8,128,1,float16,fp8,0,0.017674667139848072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,64,128,1,float16,fp8,0,0.02516799916823705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,64,128,1,float16,float16,0,0.02447466552257538
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,1,128,1,float16,float16,0,0.015263999501864115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,1,128,1,float16,fp8,0,0.01781333362062772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,2,128,1,float16,float16,0,0.015418666104475657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,2,128,1,float16,fp8,0,0.01754133279124896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,4,128,1,float16,float16,0,0.015141333142916361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,4,128,1,float16,fp8,0,0.018031999468803406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,8,128,1,float16,float16,0,0.015210667004187902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,8,128,1,float16,fp8,0,0.01752000053723653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,2,128,1,float16,fp8,0,0.017797333498795826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,64,128,1,float16,float16,0,0.01766933376590411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,64,128,1,float16,fp8,0,0.018618666877349217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,4,128,1,float16,fp8,0,0.017786666750907898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,1,128,1,float16,float16,0,0.014864000181357065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,1,128,1,float16,fp8,0,0.017535999417304993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,2,128,1,float16,float16,0,0.015082667271296183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,4,128,1,float16,float16,0,0.015381333728631338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,8,128,1,float16,float16,0,0.014991999914248785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,8,128,1,float16,fp8,0,0.016938666502634685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,64,128,1,float16,fp8,0,0.01568000018596649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,64,128,1,float16,float16,0,0.013909333695967993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,1,128,1,float16,float16,0,0.014245333770910898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,1,128,1,float16,fp8,0,0.016362667083740234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,2,128,1,float16,float16,0,0.014389333625634512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,2,128,1,float16,fp8,0,0.016362667083740234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,4,128,1,float16,float16,0,0.014432000617186228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,64,128,1,float16,fp8,0,0.014757333944241205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,4,128,1,float16,fp8,0,0.01691199963291486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,8,128,1,float16,float16,0,0.014256000518798828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,8,128,1,float16,fp8,0,0.016127999871969223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,64,128,1,float16,float16,0,0.013669333110253016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,1,128,1,float16,float16,0,0.013967999567588171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,1,128,1,float16,fp8,0,0.016202667107184727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,2,128,1,float16,float16,0,0.014192000031471252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,2,128,1,float16,fp8,0,0.01635733370979627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,4,128,1,float16,float16,0,0.014277332772811254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,4,128,1,float16,fp8,0,0.016208000481128693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,8,128,1,float16,float16,0,0.014208000153303146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,8,128,1,float16,fp8,0,0.015802666544914246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,2,128,1,float16,fp8,0,40.942395528157554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,1,128,1,float16,fp8,0,40.784741719563804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,4,128,1,float16,fp8,0,41.66442108154297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,8,128,1,float16,fp8,0,42.06363169352213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,1,128,1,float16,float16,0,45.67591857910156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,2,128,1,float16,float16,0,45.26937357584635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,4,128,1,float16,float16,0,46.06444803873698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,8,128,1,float16,float16,0,46.04374694824219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,48,128,1,float16,fp8,0,21.82702382405599
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,1,128,1,float16,fp8,0,20.65101369222005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,1,128,1,float16,float16,0,23.424697875976562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,2,128,1,float16,fp8,0,20.800399780273438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,48,128,1,float16,float16,0,24.1680908203125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,2,128,1,float16,float16,0,23.745152791341145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,4,128,1,float16,fp8,0,20.935530344645183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,4,128,1,float16,float16,0,23.26543426513672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,48,128,1,float16,fp8,0,12.14809544881185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,48,128,1,float16,float16,0,14.018421173095703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,1,128,1,float16,float16,0,14.177781422932943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,1,128,1,float16,fp8,0,12.352010091145834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,2,128,1,float16,fp8,0,11.887296040852865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,2,128,1,float16,float16,0,14.480171203613281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,48,128,1,float16,float16,0,6.522101084391276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,8,128,1,float16,fp8,0,24.00279490152995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,48,128,1,float16,fp8,0,5.754751841227214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,8,128,1,float16,float16,0,26.648885091145832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,4,128,1,float16,float16,0,11.861024220784506
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,8,128,1,float16,fp8,0,10.556581497192383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,4,128,1,float16,fp8,0,10.756479899088541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,8,128,1,float16,float16,0,12.196464538574219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,1,128,1,float16,fp8,0,5.205653190612793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,2,128,1,float16,float16,0,5.335701624552409
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,1,128,1,float16,float16,0,6.177679697672526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,2,128,1,float16,fp8,0,5.3634827931722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,4,128,1,float16,float16,0,5.751759847005208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,4,128,1,float16,fp8,0,5.29965337117513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,8,128,1,float16,float16,0,5.393856048583984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,8,128,1,float16,fp8,0,5.369498570760091
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,1,128,1,float16,fp8,0,23.39451853434245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,2,128,1,float16,fp8,0,23.49717966715495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,1,128,1,float16,float16,0,26.603001912434895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,2,128,1,float16,float16,0,26.28583526611328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,4,128,1,float16,float16,0,26.819562276204426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,4,128,1,float16,fp8,0,24.40929667154948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,8,128,1,float16,float16,0,26.735333760579426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,48,128,1,float16,float16,0,13.968037923177084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,48,128,1,float16,fp8,0,12.772378285725912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,1,128,1,float16,float16,0,13.2140261332194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,1,128,1,float16,fp8,0,11.739429473876953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,8,128,1,float16,fp8,0,24.05145009358724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,2,128,1,float16,fp8,0,11.786399841308594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,2,128,1,float16,float16,0,13.688138326009115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,4,128,1,float16,float16,0,13.328997294108072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,48,128,1,float16,float16,0,7.059557596842448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,1,128,1,float16,float16,0,6.239333470662435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,48,128,1,float16,fp8,0,6.389557520548503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,1,128,1,float16,fp8,0,6.0417226155598955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,2,128,1,float16,float16,0,6.233791987101237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,4,128,1,float16,fp8,0,12.09372329711914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,8,128,1,float16,fp8,0,12.413589477539062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,8,128,1,float16,float16,0,14.206666310628256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,2,128,1,float16,fp8,0,6.201509475708008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,4,128,1,float16,float16,0,6.390991846720378
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,48,128,1,float16,float16,0,3.2852907180786133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,4,128,1,float16,fp8,0,6.04200553894043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,48,128,1,float16,fp8,0,3.2880001068115234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,8,128,1,float16,float16,0,7.336191813151042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,1,128,1,float16,float16,0,3.2478453318277993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,1,128,1,float16,fp8,0,3.034346580505371
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,8,128,1,float16,fp8,0,6.187258402506511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,2,128,1,float16,float16,0,3.2910292943318686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,2,128,1,float16,fp8,0,3.02239990234375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,4,128,1,float16,float16,0,3.0927305221557617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,4,128,1,float16,fp8,0,3.352640151977539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,8,128,1,float16,fp8,0,3.2442506154378257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,8,128,1,float16,float16,0,3.9553600947062173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,1,128,1,float16,fp8,0,17.27245839436849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,1,128,1,float16,float16,0,19.087003072102863
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,2,128,1,float16,fp8,0,16.728858947753906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,2,128,1,float16,float16,0,18.857013702392578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,4,128,1,float16,fp8,0,17.506661732991535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,4,128,1,float16,float16,0,18.516255696614582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,8,128,1,float16,fp8,0,17.09050114949544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,8,128,1,float16,float16,0,19.302618662516277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,48,128,1,float16,float16,0,9.79525883992513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,48,128,1,float16,fp8,0,9.144586563110352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,1,128,1,float16,float16,0,10.44655990600586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,1,128,1,float16,fp8,0,8.429253260294596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,2,128,1,float16,float16,0,9.41378657023112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,2,128,1,float16,fp8,0,8.334197362263998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,4,128,1,float16,float16,0,9.81608517964681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,4,128,1,float16,fp8,0,8.746922810872396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,48,128,1,float16,float16,0,4.745583852132161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,48,128,1,float16,fp8,0,4.6243893305460615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,1,128,1,float16,float16,0,4.412325223286946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,1,128,1,float16,fp8,0,4.159237225850423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,2,128,1,float16,float16,0,4.630810737609863
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,2,128,1,float16,fp8,0,4.183717409769694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,8,128,1,float16,fp8,0,8.594522476196289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,8,128,1,float16,float16,0,9.854213078816732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,48,128,1,float16,float16,0,2.390442689259847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,4,128,1,float16,float16,0,4.585365295410156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,4,128,1,float16,fp8,0,4.29311466217041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,48,128,1,float16,fp8,0,2.3373653093973794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,8,128,1,float16,float16,0,4.78822390238444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,8,128,1,float16,fp8,0,4.314053217569987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,1,128,1,float16,float16,0,2.1443039576212564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,2,128,1,float16,float16,0,2.2118345896402993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,1,128,1,float16,fp8,0,2.567999998728434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,4,128,1,float16,float16,0,2.1758400599161782
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,2,128,1,float16,fp8,0,2.4185120264689126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,4,128,1,float16,fp8,0,2.1860639254252114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,8,128,1,float16,float16,0,2.293776035308838
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,8,128,1,float16,fp8,0,2.207477410634359
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,1,128,1,float16,fp8,0,21.720006306966145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,1,128,1,float16,float16,0,23.72523244222005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,2,128,1,float16,fp8,0,21.632001241048176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,4,128,1,float16,fp8,0,22.4105224609375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,2,128,1,float16,float16,0,23.99304962158203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,8,128,1,float16,fp8,0,22.428878784179688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,4,128,1,float16,float16,0,24.382863362630207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,8,128,1,float16,float16,0,25.134437561035156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,1,128,1,float16,float16,0,12.071999867757162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,1,128,1,float16,fp8,0,10.790475209554037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,48,128,1,float16,fp8,0,12.140042622884115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,48,128,1,float16,float16,0,14.567967732747396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,2,128,1,float16,float16,0,12.016340891520182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,2,128,1,float16,fp8,0,10.859903971354166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,4,128,1,float16,float16,0,12.669514973958334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,4,128,1,float16,fp8,0,11.568485260009766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,1,128,1,float16,float16,0,6.070314407348633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,48,128,1,float16,float16,0,6.6773331960042315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,1,128,1,float16,fp8,0,5.391679763793945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,48,128,1,float16,fp8,0,6.160874684651692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,2,128,1,float16,float16,0,6.059317270914714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,2,128,1,float16,fp8,0,5.507637023925781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,8,128,1,float16,fp8,0,11.311701456705729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,8,128,1,float16,float16,0,13.064384460449219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,48,128,1,float16,float16,0,3.2059787114461265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,4,128,1,float16,float16,0,6.320128122965495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,4,128,1,float16,fp8,0,5.61467170715332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,48,128,1,float16,fp8,0,3.078746795654297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,1,128,1,float16,float16,0,2.7394933700561523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,8,128,1,float16,float16,0,6.074917475382487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,8,128,1,float16,fp8,0,5.713781356811523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,1,128,1,float16,fp8,0,2.805216153462728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,2,128,1,float16,float16,0,2.9355945587158203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,2,128,1,float16,fp8,0,2.7436374028523765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,4,128,1,float16,float16,0,2.8289759953816733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,4,128,1,float16,fp8,0,3.0550241470336914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,8,128,1,float16,fp8,0,2.857856114705404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,48,128,1,float16,float16,0,1.607925256093343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,8,128,1,float16,float16,0,2.924266815185547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,1,128,1,float16,float16,0,1.3967679341634114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,48,128,1,float16,fp8,0,1.7318453788757324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,1,128,1,float16,fp8,0,1.4883306821187336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,2,128,1,float16,float16,0,1.4195733070373535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,2,128,1,float16,fp8,0,1.4093653361002605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,4,128,1,float16,float16,0,1.4736107190450032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,4,128,1,float16,fp8,0,1.5233440399169922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,8,128,1,float16,float16,0,1.463381290435791
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,8,128,1,float16,fp8,0,1.516965389251709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,1,128,1,float16,fp8,0,12.5230344136556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,2,128,1,float16,fp8,0,12.625311533610025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,1,128,1,float16,float16,0,15.13494873046875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,4,128,1,float16,fp8,0,13.23962656656901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,4,128,1,float16,float16,0,14.930644989013672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,8,128,1,float16,fp8,0,13.232891082763672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,8,128,1,float16,float16,0,14.448341369628906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,2,128,1,float16,float16,0,13.973871866861979
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,48,128,1,float16,float16,0,7.33958371480306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,48,128,1,float16,fp8,0,7.298661549886067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,1,128,1,float16,float16,0,6.994666417439778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,1,128,1,float16,fp8,0,6.2684370676676435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,2,128,1,float16,fp8,0,6.385440190633138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,2,128,1,float16,float16,0,7.043392181396484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,4,128,1,float16,float16,0,7.115119934082031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,4,128,1,float16,fp8,0,6.653765360514323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,48,128,1,float16,float16,0,3.6808160146077475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,1,128,1,float16,float16,0,3.2679519653320312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,48,128,1,float16,fp8,0,3.6782827377319336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,1,128,1,float16,fp8,0,3.257919947306315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,2,128,1,float16,float16,0,3.3004961013793945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,2,128,1,float16,fp8,0,3.2056214014689126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,8,128,1,float16,fp8,0,6.608757019042969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,8,128,1,float16,float16,0,7.202378590901692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,4,128,1,float16,float16,0,3.3854347864786782
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,48,128,1,float16,float16,0,1.8610773086547852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,1,128,1,float16,float16,0,1.5892319679260254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,4,128,1,float16,fp8,0,3.6410932540893555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,48,128,1,float16,fp8,0,1.8770826657613118
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,8,128,1,float16,float16,0,3.3579788208007812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,1,128,1,float16,fp8,0,1.7120320002237956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,8,128,1,float16,fp8,0,3.368218739827474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,2,128,1,float16,float16,0,1.7016587257385254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,4,128,1,float16,float16,0,1.672719955444336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,4,128,1,float16,fp8,0,1.7130346298217773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,2,128,1,float16,fp8,0,1.6379307111104329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,48,128,1,float16,float16,0,0.9525706768035889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,8,128,1,float16,float16,0,1.6898667017618816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,48,128,1,float16,fp8,0,0.9796266555786133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,8,128,1,float16,fp8,0,1.7809173266092937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,1,128,1,float16,float16,0,0.870965321858724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,1,128,1,float16,fp8,0,0.8571146329243978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,2,128,1,float16,float16,0,0.8326293627421061
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,2,128,1,float16,fp8,0,0.8597066402435303
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,4,128,1,float16,float16,0,0.8467199802398682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,8,128,1,float16,float16,0,0.8768479824066162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,4,128,1,float16,fp8,0,0.9147093296051025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,8,128,1,float16,fp8,0,0.9239466985066732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,1,128,1,float16,float16,0,12.980475107828775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,1,128,1,float16,fp8,0,12.0240109761556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,2,128,1,float16,float16,0,13.43020248413086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,2,128,1,float16,fp8,0,12.062405904134115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,4,128,1,float16,float16,0,13.893941243489584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,4,128,1,float16,fp8,0,13.05074691772461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,8,128,1,float16,fp8,0,12.764565785725912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,8,128,1,float16,float16,0,13.750447591145834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,1,128,1,float16,float16,0,6.183866500854492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,1,128,1,float16,fp8,0,5.903514862060547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,48,128,1,float16,float16,0,7.3094024658203125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,2,128,1,float16,float16,0,6.434069315592448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,48,128,1,float16,fp8,0,7.239168167114258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,2,128,1,float16,fp8,0,6.022970835367839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,4,128,1,float16,float16,0,6.584362665812175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,4,128,1,float16,fp8,0,6.4826507568359375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,1,128,1,float16,float16,0,2.9392693837483725
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,48,128,1,float16,float16,0,3.6549386978149414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,1,128,1,float16,fp8,0,3.00057061513265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,48,128,1,float16,fp8,0,3.9167839686075845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,2,128,1,float16,float16,0,3.091279983520508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,8,128,1,float16,float16,0,6.805871963500977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,8,128,1,float16,fp8,0,6.363861083984375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,2,128,1,float16,fp8,0,3.0446879069010415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,48,128,1,float16,float16,0,1.8519946734110515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,4,128,1,float16,float16,0,3.2183094024658203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,8,128,1,float16,float16,0,3.243333180745443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,8,128,1,float16,fp8,0,3.364890734354655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,4,128,1,float16,fp8,0,3.4478238423665366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,48,128,1,float16,fp8,0,1.8516693115234375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,1,128,1,float16,float16,0,1.4907466570536296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,2,128,1,float16,float16,0,1.5427039464314778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,1,128,1,float16,fp8,0,1.5849599838256836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,2,128,1,float16,fp8,0,1.6366346677144368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,4,128,1,float16,float16,0,1.5750400225321453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,4,128,1,float16,fp8,0,1.6806294123331706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,8,128,1,float16,float16,0,1.5968799591064453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,48,128,1,float16,float16,0,0.933456023534139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,8,128,1,float16,fp8,0,1.7079946200052898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,1,128,1,float16,float16,0,0.775322675704956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,48,128,1,float16,fp8,0,0.9872960249582926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,1,128,1,float16,fp8,0,0.7854506969451904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,2,128,1,float16,float16,0,0.7771680355072021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,4,128,1,float16,float16,0,0.8086506525675455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,2,128,1,float16,fp8,0,0.7891733646392822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,4,128,1,float16,fp8,0,0.8637119928995768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,48,128,1,float16,float16,0,0.4808533191680908
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,8,128,1,float16,float16,0,0.8366826375325521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,48,128,1,float16,fp8,0,0.5218720038731893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,8,128,1,float16,fp8,0,0.8797439734141032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,1,128,1,float16,float16,0,0.3935626745223999
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,2,128,1,float16,float16,0,0.40595734119415283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,1,128,1,float16,fp8,0,0.4109333356221517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,2,128,1,float16,fp8,0,0.43243201573689777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,8,128,1,float16,float16,0,0.43408532937367755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,4,128,1,float16,float16,0,0.4286880095799764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,4,128,1,float16,fp8,0,0.4549493392308553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,8,128,1,float16,fp8,0,0.4532533486684163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,1,128,1,float16,float16,0,7.69276237487793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,2,128,1,float16,float16,0,7.348042805989583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,2,128,1,float16,fp8,0,7.354442596435547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,1,128,1,float16,fp8,0,7.242144266764323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,4,128,1,float16,float16,0,7.839743932088216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,4,128,1,float16,fp8,0,7.849285125732422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,8,128,1,float16,float16,0,8.002949396769205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,1,128,1,float16,float16,0,3.517925262451172
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,1,128,1,float16,fp8,0,3.599029223124186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,2,128,1,float16,float16,0,3.6966613133748374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,48,128,1,float16,float16,0,4.50764815012614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,2,128,1,float16,fp8,0,3.633359909057617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,48,128,1,float16,fp8,0,4.773344039916992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,4,128,1,float16,float16,0,4.144309361775716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,1,128,1,float16,float16,0,1.7239467302958171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,48,128,1,float16,float16,0,2.2511679331461587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,48,128,1,float16,fp8,0,2.3144373893737793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,4,128,1,float16,fp8,0,4.116575876871745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,1,128,1,float16,fp8,0,1.8379039764404297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,8,128,1,float16,fp8,0,7.859423955281575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,8,128,1,float16,float16,0,3.8173866271972656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,8,128,1,float16,fp8,0,4.076944033304851
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,2,128,1,float16,float16,0,1.7779199282328289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,4,128,1,float16,float16,0,1.8830666542053223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,2,128,1,float16,fp8,0,1.8772907257080078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,4,128,1,float16,fp8,0,1.9918665885925293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,8,128,1,float16,float16,0,1.9059732755025227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,48,128,1,float16,fp8,0,1.1914506753285725
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,1,128,1,float16,float16,0,0.8872693379720052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,48,128,1,float16,float16,0,1.1431146462758381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,1,128,1,float16,fp8,0,0.9584000110626221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,8,128,1,float16,fp8,0,2.0073067347208657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,2,128,1,float16,float16,0,0.9202400048573812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,2,128,1,float16,fp8,0,0.9399200280507406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,4,128,1,float16,float16,0,0.9872960249582926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,4,128,1,float16,fp8,0,1.0196639696757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,8,128,1,float16,fp8,0,1.054965337117513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,8,128,1,float16,float16,0,0.9755307038625082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,48,128,1,float16,float16,0,0.586517333984375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,48,128,1,float16,fp8,0,0.6200000047683716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,1,128,1,float16,float16,0,0.46064531803131104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,1,128,1,float16,fp8,0,0.48279468218485516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,2,128,1,float16,float16,0,0.47297600905100506
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,2,128,1,float16,fp8,0,0.49094398816426593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,4,128,1,float16,float16,0,0.49720533688863117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,4,128,1,float16,fp8,0,0.5406560103098551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,8,128,1,float16,float16,0,0.5047039985656738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,8,128,1,float16,fp8,0,0.5363733371098837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,48,128,1,float16,float16,0,0.3174506624539693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,48,128,1,float16,fp8,0,0.3333973487218221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,1,128,1,float16,float16,0,0.24501333634058634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,1,128,1,float16,fp8,0,0.25971200068791706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,2,128,1,float16,fp8,0,0.2723146677017212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,2,128,1,float16,float16,0,0.2534186641375224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,4,128,1,float16,float16,0,0.26500799258550006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,4,128,1,float16,fp8,0,0.2845120032628377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,8,128,1,float16,float16,0,0.2728533347447713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,8,128,1,float16,fp8,0,0.29097066322962445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,1,128,1,float16,float16,0,7.264074961344401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,1,128,1,float16,fp8,0,7.065856297810872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,2,128,1,float16,float16,0,7.295930862426758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,4,128,1,float16,fp8,0,7.9919999440511065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,4,128,1,float16,float16,0,7.617925643920898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,8,128,1,float16,float16,0,7.781061172485352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,8,128,1,float16,fp8,0,7.944474538167317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,1,128,1,float16,float16,0,3.402799924214681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,48,128,1,float16,float16,0,4.660117467244466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,48,128,1,float16,fp8,0,4.888245264689128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,1,128,1,float16,fp8,0,3.5289653142293296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,2,128,1,float16,float16,0,3.471877415974935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,2,128,1,float16,fp8,0,3.6258932749430337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,4,128,1,float16,float16,0,3.8028265635172525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,2,128,1,float16,fp8,0,7.224224090576172
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,8,128,1,float16,float16,0,3.855072021484375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,48,128,1,float16,float16,0,2.3305439949035645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,4,128,1,float16,fp8,0,4.115711847941081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,48,128,1,float16,fp8,0,2.464757283528646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,8,128,1,float16,fp8,0,3.979328155517578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,1,128,1,float16,float16,0,1.6804639498392742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,2,128,1,float16,float16,0,1.7812213897705078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,1,128,1,float16,fp8,0,1.7936426798502605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,4,128,1,float16,fp8,0,2.0196053187052407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,4,128,1,float16,float16,0,1.9067413012186687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,2,128,1,float16,fp8,0,1.8344000180562336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,8,128,1,float16,fp8,0,2.0605546633402505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,8,128,1,float16,float16,0,1.932528018951416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,48,128,1,float16,float16,0,1.2059626579284668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,48,128,1,float16,fp8,0,1.2595946788787842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,1,128,1,float16,float16,0,0.8550293445587158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,2,128,1,float16,float16,0,0.897536039352417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,1,128,1,float16,fp8,0,0.9335199991861979
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,2,128,1,float16,fp8,0,0.9499039649963379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,4,128,1,float16,float16,0,0.9614026546478271
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,8,128,1,float16,float16,0,0.9964373111724854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,48,128,1,float16,float16,0,0.6040106614430746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,8,128,1,float16,fp8,0,1.0405866305033367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,48,128,1,float16,fp8,0,0.6649653514226278
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,1,128,1,float16,float16,0,0.4448586702346802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,4,128,1,float16,fp8,0,1.0650453567504883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,1,128,1,float16,fp8,0,0.4843413432439168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,2,128,1,float16,float16,0,0.4551946719487508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,2,128,1,float16,fp8,0,0.49351465702056885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,4,128,1,float16,fp8,0,0.5329440037409464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,4,128,1,float16,float16,0,0.49293867746988934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,8,128,1,float16,float16,0,0.5068639914194742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,8,128,1,float16,fp8,0,0.555178682009379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,48,128,1,float16,float16,0,0.325872004032135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,48,128,1,float16,fp8,0,0.3500426610310872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,1,128,1,float16,float16,0,0.23312000433603922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,1,128,1,float16,fp8,0,0.25913067658742267
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,2,128,1,float16,float16,0,0.23966399828592935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,2,128,1,float16,fp8,0,0.2632746696472168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,4,128,1,float16,float16,0,0.2656960090001424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,4,128,1,float16,fp8,0,0.28860267003377277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,8,128,1,float16,float16,0,0.2710453271865845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,8,128,1,float16,fp8,0,0.29056533177693683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,48,128,1,float16,float16,0,0.17323732376098633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,48,128,1,float16,fp8,0,0.1938719948132833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,1,128,1,float16,float16,0,0.12738666931788126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,1,128,1,float16,fp8,0,0.14289599657058716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,2,128,1,float16,float16,0,0.13411733508110046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,2,128,1,float16,fp8,0,0.14757333199183145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,4,128,1,float16,float16,0,0.14662399888038635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,4,128,1,float16,fp8,0,0.1588373382886251
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,8,128,1,float16,float16,0,0.15092800060908
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,8,128,1,float16,fp8,0,0.1644426683584849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,1,128,1,float16,float16,0,4.199701309204102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,1,128,1,float16,fp8,0,4.457306543986003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,2,128,1,float16,float16,0,4.251242637634277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,2,128,1,float16,fp8,0,4.534346580505371
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,4,128,1,float16,float16,0,4.756165186564128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,8,128,1,float16,float16,0,4.955077489217122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,4,128,1,float16,fp8,0,5.223957379659017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,8,128,1,float16,fp8,0,5.314607938130696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,48,128,1,float16,float16,0,3.1335681279500327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,1,128,1,float16,float16,0,2.0943466822306314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,2,128,1,float16,float16,0,2.2192373275756836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,48,128,1,float16,fp8,0,3.2654987970987954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,1,128,1,float16,fp8,0,2.2275306383768716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,4,128,1,float16,float16,0,2.3414079348246255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,2,128,1,float16,fp8,0,2.380575974782308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,4,128,1,float16,fp8,0,2.584869384765625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,1,128,1,float16,float16,0,1.0315146446228027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,48,128,1,float16,fp8,0,1.6529173851013184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,8,128,1,float16,float16,0,2.4209973017374673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,8,128,1,float16,fp8,0,2.647439956665039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,48,128,1,float16,float16,0,1.529695987701416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,1,128,1,float16,fp8,0,1.1287253697713215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,2,128,1,float16,float16,0,1.1134719848632812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,2,128,1,float16,fp8,0,1.1604106426239014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,4,128,1,float16,float16,0,1.1832533677419026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,4,128,1,float16,fp8,0,1.3010986646016438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,48,128,1,float16,float16,0,0.7794346809387207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,8,128,1,float16,float16,0,1.232415994008382
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,8,128,1,float16,fp8,0,1.348197301228841
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,48,128,1,float16,fp8,0,0.8454933166503906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,1,128,1,float16,float16,0,0.5387839873631796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,1,128,1,float16,fp8,0,0.5798933506011963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,2,128,1,float16,float16,0,0.5517333348592123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,4,128,1,float16,float16,0,0.6222879886627197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,2,128,1,float16,fp8,0,0.5927733182907104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,4,128,1,float16,fp8,0,0.6926773389180502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,8,128,1,float16,float16,0,0.6149866580963135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,8,128,1,float16,fp8,0,0.6954240004221598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,48,128,1,float16,fp8,0,0.44087998072306317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,48,128,1,float16,float16,0,0.408896009127299
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,1,128,1,float16,float16,0,0.27505600452423096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,1,128,1,float16,fp8,0,0.31426666180292767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,2,128,1,float16,float16,0,0.2906879981358846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,2,128,1,float16,fp8,0,0.3199839989344279
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,4,128,1,float16,float16,0,0.3173813422520955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,4,128,1,float16,fp8,0,0.3500053485234578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,8,128,1,float16,float16,0,0.3308853308359782
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,8,128,1,float16,fp8,0,0.3557120164235433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,48,128,1,float16,float16,0,0.22043200333913168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,48,128,1,float16,fp8,0,0.24031466245651245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,1,128,1,float16,float16,0,0.15441600481669107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,1,128,1,float16,fp8,0,0.1694986621538798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,2,128,1,float16,float16,0,0.1593226691087087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,4,128,1,float16,float16,0,0.17012800772984824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,2,128,1,float16,fp8,0,0.17612266540527344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,8,128,1,float16,float16,0,0.17826666434605917
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,4,128,1,float16,fp8,0,0.19486399491628012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,8,128,1,float16,fp8,0,0.19505600134531656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,48,128,1,float16,float16,0,0.12086932857831319
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,48,128,1,float16,fp8,0,0.1355893313884735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,1,128,1,float16,float16,0,0.08861866593360901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,1,128,1,float16,fp8,0,0.09738133351008098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,2,128,1,float16,float16,0,0.08927999933560689
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,4,128,1,float16,float16,0,0.09735467036565144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,2,128,1,float16,fp8,0,0.09864532947540283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,8,128,1,float16,float16,0,0.09905067086219788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,4,128,1,float16,fp8,0,0.10912000139554341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,8,128,1,float16,fp8,0,0.11181333661079407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,1,128,1,float16,fp8,0,4.754597345987956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,2,128,1,float16,float16,0,4.580138524373372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,4,128,1,float16,float16,0,5.133280118306478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,8,128,1,float16,float16,0,5.156725247701009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,1,128,1,float16,float16,0,4.434629440307617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,2,128,1,float16,fp8,0,4.878879865010579
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,1,128,1,float16,float16,0,2.1587146123250327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,48,128,1,float16,float16,0,3.399050712585449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,8,128,1,float16,fp8,0,5.615002950032552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,48,128,1,float16,fp8,0,3.8847573598225913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,1,128,1,float16,fp8,0,2.368586699167887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,4,128,1,float16,fp8,0,5.713653564453125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,2,128,1,float16,fp8,0,2.4559733072916665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,4,128,1,float16,float16,0,2.550271987915039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,8,128,1,float16,float16,0,2.545248031616211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,4,128,1,float16,fp8,0,2.8606719970703125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,48,128,1,float16,float16,0,1.7150986989339192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,1,128,1,float16,float16,0,1.0763146877288818
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,8,128,1,float16,fp8,0,2.848170598347982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,1,128,1,float16,fp8,0,1.1990559895833333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,2,128,1,float16,float16,0,1.138271967569987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,2,128,1,float16,fp8,0,1.2451732953389485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,8,128,1,float16,float16,0,1.3256213665008545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,48,128,1,float16,fp8,0,1.9266773859659831
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,4,128,1,float16,float16,0,1.3077759742736816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,4,128,1,float16,fp8,0,1.4347947438557942
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,48,128,1,float16,float16,0,0.8698293368021647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,8,128,1,float16,fp8,0,1.4747625986735027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,2,128,1,float16,float16,0,2.289600054423014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,1,128,1,float16,float16,0,0.5634880065917969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,1,128,1,float16,fp8,0,0.6133013168970743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,48,128,1,float16,fp8,0,0.9890346527099609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,2,128,1,float16,float16,0,0.6039093335469564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,2,128,1,float16,fp8,0,0.6339199940363566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,4,128,1,float16,fp8,0,0.7354880174001058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,4,128,1,float16,float16,0,0.6640586853027344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,8,128,1,float16,fp8,0,0.765775998433431
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,48,128,1,float16,float16,0,0.44280532995859784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,48,128,1,float16,fp8,0,0.5065813461939493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,1,128,1,float16,float16,0,0.29444267352422077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,1,128,1,float16,fp8,0,0.3203999996185303
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,8,128,1,float16,float16,0,0.6668372948964437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,2,128,1,float16,float16,0,0.30268800258636475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,2,128,1,float16,fp8,0,0.3357066710789998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,4,128,1,float16,float16,0,0.3495306571324666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,8,128,1,float16,float16,0,0.34863468011220294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,4,128,1,float16,fp8,0,0.3816053469975789
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,48,128,1,float16,float16,0,0.23516800006230673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,8,128,1,float16,fp8,0,0.39946667353312176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,1,128,1,float16,float16,0,0.15370666980743408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,48,128,1,float16,fp8,0,0.27938133478164673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,1,128,1,float16,fp8,0,0.17587200800577799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,2,128,1,float16,float16,0,0.16687999169031778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,2,128,1,float16,fp8,0,0.1844266653060913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,8,128,1,float16,fp8,0,0.20860799153645834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,4,128,1,float16,float16,0,0.18238399426142374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,8,128,1,float16,float16,0,0.18920532862345377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,48,128,1,float16,fp8,0,0.15121600031852722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,1,128,1,float16,float16,0,0.09011733531951904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,4,128,1,float16,fp8,0,0.208624005317688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,1,128,1,float16,fp8,0,0.10228799780209859
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,48,128,1,float16,float16,0,0.13084800044695535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,2,128,1,float16,float16,0,0.09057066837946574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,2,128,1,float16,fp8,0,0.10397866368293762
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,4,128,1,float16,float16,0,0.10173333684603374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,4,128,1,float16,fp8,0,0.11567466457684834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,8,128,1,float16,float16,0,0.10577066739400227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,8,128,1,float16,fp8,0,0.11944533387819926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,48,128,1,float16,float16,0,0.07603733241558075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,48,128,1,float16,fp8,0,0.08922666311264038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,1,128,1,float16,float16,0,0.04740266501903534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,1,128,1,float16,fp8,0,0.0544106662273407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,2,128,1,float16,float16,0,0.05197866757710775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,2,128,1,float16,fp8,0,0.05885333319505056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,4,128,1,float16,float16,0,0.05570666491985321
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,4,128,1,float16,fp8,0,0.06419200201829274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,8,128,1,float16,float16,0,0.05904533465703329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,8,128,1,float16,fp8,0,0.06408533453941345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,1,128,1,float16,float16,0,3.1294078826904297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,2,128,1,float16,float16,0,3.4184160232543945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,1,128,1,float16,fp8,0,3.523717244466146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,2,128,1,float16,fp8,0,3.7594614028930664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,4,128,1,float16,float16,0,3.9956960678100586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,8,128,1,float16,float16,0,3.9995787938435874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,1,128,1,float16,float16,0,1.5754399299621582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,48,128,1,float16,float16,0,2.8295466105143228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,48,128,1,float16,fp8,0,3.3912585576375327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,1,128,1,float16,fp8,0,1.7727573712666829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,8,128,1,float16,fp8,0,4.579781214396159
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,4,128,1,float16,fp8,0,4.59658145904541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,2,128,1,float16,fp8,0,1.8787946701049805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,4,128,1,float16,float16,0,1.9967519442240398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,4,128,1,float16,fp8,0,2.305253346761068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,1,128,1,float16,float16,0,0.7982239723205566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,2,128,1,float16,float16,0,1.7022825876871746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,8,128,1,float16,fp8,0,2.3184266090393066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,2,128,1,float16,fp8,0,0.9510826269785563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,48,128,1,float16,fp8,0,1.7093173662821453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,1,128,1,float16,fp8,0,0.9009280204772949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,4,128,1,float16,float16,0,1.003226677576701
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,2,128,1,float16,float16,0,0.860485315322876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,48,128,1,float16,float16,0,1.4235572814941406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,48,128,1,float16,float16,0,0.7378613154093424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,8,128,1,float16,float16,0,1.0223039786020915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,1,128,1,float16,float16,0,0.41150931517283124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,1,128,1,float16,fp8,0,0.46425068378448486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,48,128,1,float16,fp8,0,0.8751306533813477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,4,128,1,float16,fp8,0,1.1637866497039795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,8,128,1,float16,fp8,0,1.2071306705474854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,2,128,1,float16,float16,0,0.4471520185470581
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,4,128,1,float16,float16,0,0.5133333206176758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,2,128,1,float16,fp8,0,0.4896533489227295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,8,128,1,float16,float16,0,2.0118080774943032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,8,128,1,float16,float16,0,0.5228639841079712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,8,128,1,float16,fp8,0,0.6156853437423706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,4,128,1,float16,fp8,0,0.5949653387069702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,48,128,1,float16,fp8,0,0.4524853229522705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,48,128,1,float16,float16,0,0.38078399499257404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,1,128,1,float16,float16,0,0.216048002243042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,1,128,1,float16,fp8,0,0.2437866727511088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,2,128,1,float16,fp8,0,0.25861332813898724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,4,128,1,float16,float16,0,0.2674880027770996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,4,128,1,float16,fp8,0,0.3096160093943278
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,2,128,1,float16,float16,0,0.235152006149292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,48,128,1,float16,float16,0,0.1976213256518046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,1,128,1,float16,fp8,0,0.13478400309880575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,48,128,1,float16,fp8,0,0.2376533349355062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,8,128,1,float16,fp8,0,0.322160005569458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,1,128,1,float16,float16,0,0.11819199721018474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,8,128,1,float16,float16,0,0.2715733249982198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,2,128,1,float16,float16,0,0.12623467048009238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,8,128,1,float16,fp8,0,0.17314666509628296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,2,128,1,float16,fp8,0,0.143696000178655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,4,128,1,float16,float16,0,0.1420693298180898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,4,128,1,float16,fp8,0,0.16547733545303345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,8,128,1,float16,float16,0,0.1451359987258911
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,48,128,1,float16,float16,0,0.10851732889811198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,48,128,1,float16,fp8,0,0.13221866885821024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,1,128,1,float16,float16,0,0.06828799843788147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,1,128,1,float16,fp8,0,0.07934933404127757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,2,128,1,float16,float16,0,0.0717493345340093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,4,128,1,float16,float16,0,0.0796853353579839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,2,128,1,float16,fp8,0,0.08349866668383281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,4,128,1,float16,fp8,0,0.09265066186587016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,8,128,1,float16,float16,0,0.08159466584523518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,8,128,1,float16,fp8,0,0.09743466973304749
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,48,128,1,float16,float16,0,0.06346666812896729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,48,128,1,float16,fp8,0,0.07787733276685078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,1,128,1,float16,float16,0,0.035760000348091125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,1,128,1,float16,fp8,0,0.04324266811211904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,2,128,1,float16,float16,0,0.040394666294256844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,2,128,1,float16,fp8,0,0.045909335215886436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,4,128,1,float16,float16,0,0.042170668641726174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,4,128,1,float16,fp8,0,0.05053333441416422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,8,128,1,float16,float16,0,0.04653333127498627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,8,128,1,float16,fp8,0,0.05117866893609365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,48,128,1,float16,float16,0,0.03692800054947535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,48,128,1,float16,fp8,0,0.04339733223120371
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,1,128,1,float16,float16,0,0.02699200063943863
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,1,128,1,float16,fp8,0,0.03017599880695343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,2,128,1,float16,float16,0,0.027189334233601887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,2,128,1,float16,fp8,0,0.030218665798505146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,4,128,1,float16,float16,0,0.02809600035349528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,4,128,1,float16,fp8,0,0.03369066615899404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,8,128,1,float16,float16,0,0.028149334092934925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,8,128,1,float16,fp8,0,0.03477866699298223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,1,128,1,float16,fp8,0,1.4799572626749675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,1,128,1,float16,float16,0,1.3120960394541423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,2,128,1,float16,fp8,0,1.6131466229756672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,2,128,1,float16,float16,0,1.4475253423055012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,48,128,1,float16,float16,0,1.3096693356831868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,4,128,1,float16,fp8,0,2.023893356323242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,1,128,1,float16,float16,0,0.6657546758651733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,8,128,1,float16,float16,0,1.7394506136576335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,48,128,1,float16,fp8,0,1.4849066734313965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,4,128,1,float16,float16,0,1.7243893941243489
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,8,128,1,float16,fp8,0,2.0540159543355307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,1,128,1,float16,fp8,0,0.7513013680775961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,2,128,1,float16,fp8,0,0.8189226786295573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,2,128,1,float16,float16,0,0.7240426540374756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,8,128,1,float16,float16,0,0.8904746373494467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,48,128,1,float16,float16,0,0.6645493507385254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,4,128,1,float16,fp8,0,1.0240320364634197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,4,128,1,float16,float16,0,0.8767733573913574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,1,128,1,float16,float16,0,0.34434668223063153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,48,128,1,float16,fp8,0,0.7553439935048422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,8,128,1,float16,fp8,0,1.0644480387369792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,2,128,1,float16,float16,0,0.3771359920501709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,1,128,1,float16,fp8,0,0.39052267869313556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,2,128,1,float16,fp8,0,0.4240586757659912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,4,128,1,float16,float16,0,0.4479573170344035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,4,128,1,float16,fp8,0,0.5226133267084757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,8,128,1,float16,float16,0,0.4554080168406169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,48,128,1,float16,float16,0,0.34322134653727215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,8,128,1,float16,fp8,0,0.5426079829533895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,48,128,1,float16,fp8,0,0.39189334710439044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,1,128,1,float16,float16,0,0.18244266510009766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,1,128,1,float16,fp8,0,0.20784000555674234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,2,128,1,float16,float16,0,0.19792532920837402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,2,128,1,float16,fp8,0,0.22220800320307413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,4,128,1,float16,float16,0,0.23197333017985025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,4,128,1,float16,fp8,0,0.27314666906992596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,8,128,1,float16,float16,0,0.23732266823450723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,8,128,1,float16,fp8,0,0.2863039970397949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,48,128,1,float16,float16,0,0.18390933672587076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,2,128,1,float16,float16,0,0.10790933171908061
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,48,128,1,float16,fp8,0,0.20986133813858032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,1,128,1,float16,float16,0,0.10108266274134318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,1,128,1,float16,fp8,0,0.11385066310564677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,2,128,1,float16,fp8,0,0.12392533818880717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,4,128,1,float16,float16,0,0.125408003727595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,8,128,1,float16,float16,0,0.1276533305644989
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,48,128,1,float16,float16,0,0.1011253297328949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,4,128,1,float16,fp8,0,0.1486240029335022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,48,128,1,float16,fp8,0,0.11693867047627766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,1,128,1,float16,float16,0,0.058058664202690125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,8,128,1,float16,fp8,0,0.15520000457763672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,2,128,1,float16,float16,0,0.062368000547091164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,2,128,1,float16,fp8,0,0.07156800230344136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,1,128,1,float16,fp8,0,0.06791999936103821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,4,128,1,float16,float16,0,0.07045333087444305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,4,128,1,float16,fp8,0,0.08477866649627686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,8,128,1,float16,float16,0,0.07260266443093617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,8,128,1,float16,fp8,0,0.08659733335177104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,48,128,1,float16,float16,0,0.059104000528653465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,2,128,1,float16,fp8,0,0.03856533269087473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,1,128,1,float16,float16,0,0.03124266614516576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,48,128,1,float16,fp8,0,0.06867200136184692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,1,128,1,float16,fp8,0,0.036042665441830955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,2,128,1,float16,float16,0,0.03244800120592117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,4,128,1,float16,float16,0,0.03565333286921183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,4,128,1,float16,fp8,0,0.04181333382924398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,1,128,1,float16,float16,0,0.02233600119749705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,48,128,1,float16,float16,0,0.031530665854612984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,8,128,1,float16,float16,0,0.0391839991013209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,8,128,1,float16,fp8,0,0.045909335215886436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,48,128,1,float16,fp8,0,0.03677333394686381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,1,128,1,float16,fp8,0,0.026704000929991405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,2,128,1,float16,float16,0,0.022709332406520844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,2,128,1,float16,fp8,0,0.02625600000222524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,4,128,1,float16,float16,0,0.02365333338578542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,4,128,1,float16,fp8,0,0.02812266598145167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,8,128,1,float16,float16,0,0.024085332949956257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,8,128,1,float16,fp8,0,0.02951466788848241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,48,128,1,float16,float16,0,0.01904533306757609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,48,128,1,float16,fp8,0,0.023530667026837666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,1,128,1,float16,float16,0,0.01695999999841054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,1,128,1,float16,fp8,0,0.01974933346112569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,2,128,1,float16,float16,0,0.017157333592573803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,2,128,1,float16,fp8,0,0.020106667031844456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,4,128,1,float16,float16,0,0.017504000415404636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,4,128,1,float16,fp8,0,0.020341333001852036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,8,128,1,float16,float16,0,0.017338667064905167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,8,128,1,float16,fp8,0,0.020928000410397846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,1,128,1,float16,float16,0,0.6385013262430826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,1,128,1,float16,fp8,0,0.6835839748382568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,2,128,1,float16,float16,0,0.7136586507161459
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,2,128,1,float16,fp8,0,0.7309280236562093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,48,128,1,float16,float16,0,0.6387840112050375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,4,128,1,float16,float16,0,0.8487679958343506
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,48,128,1,float16,fp8,0,0.6890772978464762
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,8,128,1,float16,float16,0,0.8629973729451498
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,8,128,1,float16,fp8,0,0.9716906547546387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,4,128,1,float16,fp8,0,0.9473120371500651
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,1,128,1,float16,float16,0,0.33052267630894977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,1,128,1,float16,fp8,0,0.3548853397369385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,2,128,1,float16,float16,0,0.36453866958618164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,2,128,1,float16,fp8,0,0.38493335247039795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,4,128,1,float16,float16,0,0.430458664894104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,4,128,1,float16,fp8,0,0.4819733301798503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,8,128,1,float16,float16,0,0.4391893148422241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,48,128,1,float16,fp8,0,0.35628799597422284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,1,128,1,float16,float16,0,0.17323199907938638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,48,128,1,float16,float16,0,0.3293866713841756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,1,128,1,float16,fp8,0,0.19345066944758096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,8,128,1,float16,fp8,0,0.5027199983596802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,2,128,1,float16,float16,0,0.19125332434972128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,2,128,1,float16,fp8,0,0.2021440068880717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,4,128,1,float16,float16,0,0.22283732891082764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,4,128,1,float16,fp8,0,0.25496000051498413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,8,128,1,float16,float16,0,0.22844266891479492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,8,128,1,float16,fp8,0,0.25990933179855347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,48,128,1,float16,fp8,0,0.19156799713770548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,1,128,1,float16,float16,0,0.09382933378219604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,48,128,1,float16,float16,0,0.17693867286046347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,1,128,1,float16,fp8,0,0.10869333148002625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,2,128,1,float16,float16,0,0.10313066840171814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,4,128,1,float16,float16,0,0.11940266688664754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,2,128,1,float16,fp8,0,0.11601066589355469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,4,128,1,float16,fp8,0,0.13889599839846292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,8,128,1,float16,float16,0,0.1222879985968272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,48,128,1,float16,float16,0,0.09819733103116353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,48,128,1,float16,fp8,0,0.10587199529012044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,8,128,1,float16,fp8,0,0.14453333616256714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,1,128,1,float16,float16,0,0.05533866584300995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,1,128,1,float16,fp8,0,0.062463998794555664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,2,128,1,float16,float16,0,0.059061333537101746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,2,128,1,float16,fp8,0,0.06559466818968455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,4,128,1,float16,float16,0,0.0679253339767456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,4,128,1,float16,fp8,0,0.07760533193747203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,8,128,1,float16,float16,0,0.06966400146484375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,8,128,1,float16,fp8,0,0.07937066753705342
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,48,128,1,float16,float16,0,0.05598400036493937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,48,128,1,float16,fp8,0,0.05951466659704844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,1,128,1,float16,float16,0,0.0273333340883255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,1,128,1,float16,fp8,0,0.03200533241033554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,4,128,1,float16,fp8,0,0.03664000084002813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,8,128,1,float16,float16,0,0.03495466709136963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,2,128,1,float16,float16,0,0.02886933336655299
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,2,128,1,float16,fp8,0,0.03331200033426285
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,4,128,1,float16,float16,0,0.03266666581233343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,8,128,1,float16,fp8,0,0.03775466730197271
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,48,128,1,float16,float16,0,0.03203733265399933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,48,128,1,float16,fp8,0,0.030666666726271313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,1,128,1,float16,float16,0,0.02085866779088974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,1,128,1,float16,fp8,0,0.024341332415739696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,2,128,1,float16,float16,0,0.021231998999913532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,2,128,1,float16,fp8,0,0.024330665667851765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,4,128,1,float16,float16,0,0.022277332842350006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,8,128,1,float16,float16,0,0.022533332308133442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,8,128,1,float16,fp8,0,0.02611200014750163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,48,128,1,float16,float16,0,0.01739199956258138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,2,128,1,float16,fp8,0,0.01810666670401891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,4,128,1,float16,fp8,0,0.025973332424958546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,48,128,1,float16,fp8,0,0.020021333048741024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,1,128,1,float16,float16,0,0.015599999576807022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,1,128,1,float16,fp8,0,0.017690667261679966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,48,128,1,float16,float16,0,0.016437333077192307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,2,128,1,float16,float16,0,0.015509333461523056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,4,128,1,float16,float16,0,0.015781333049138386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,4,128,1,float16,fp8,0,0.017952000101407368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,2,128,1,float16,fp8,0,0.017973333597183228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,8,128,1,float16,float16,0,0.015919999529918034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,8,128,1,float16,fp8,0,0.018122666825850803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,48,128,1,float16,fp8,0,0.01929066702723503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,1,128,1,float16,float16,0,0.015562667200962702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,1,128,1,float16,fp8,0,0.017429333180189133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,2,128,1,float16,float16,0,0.01523200049996376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,4,128,1,float16,float16,0,0.015103999525308609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,4,128,1,float16,fp8,0,0.01800000046690305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,8,128,1,float16,float16,0,0.015360000232855478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,8,128,1,float16,fp8,0,0.017680000513792038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,1,128,1,float16,float16,0,0.32838932673136395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,1,128,1,float16,fp8,0,0.3581920067469279
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,2,128,1,float16,float16,0,0.3645919958750407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,2,128,1,float16,fp8,0,0.38045867284138996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,48,128,1,float16,fp8,0,0.3607626756032308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,48,128,1,float16,float16,0,0.3688053290049235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,8,128,1,float16,fp8,0,0.4931199947992961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,4,128,1,float16,float16,0,0.42719467480977374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,1,128,1,float16,float16,0,0.17313599586486816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,8,128,1,float16,float16,0,0.4389439821243286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,4,128,1,float16,fp8,0,0.4809066851933797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,1,128,1,float16,fp8,0,0.19096000989278158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,8,128,1,float16,float16,0,0.2290666699409485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,48,128,1,float16,fp8,0,0.1894773244857788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,48,128,1,float16,float16,0,0.19242133696873984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,4,128,1,float16,float16,0,0.22256000836690268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,4,128,1,float16,fp8,0,0.2523733377456665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,1,128,1,float16,float16,0,0.09564266602198283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,8,128,1,float16,fp8,0,0.2621120015780131
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,2,128,1,float16,float16,0,0.18995199600855509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,2,128,1,float16,fp8,0,0.20465600490570068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,1,128,1,float16,fp8,0,0.1088266670703888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,4,128,1,float16,fp8,0,0.13904000322024027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,2,128,1,float16,fp8,0,0.11571199695269267
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,8,128,1,float16,float16,0,0.12307733297348022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,4,128,1,float16,float16,0,0.11868799726168315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,48,128,1,float16,fp8,0,0.10229866703351338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,48,128,1,float16,float16,0,0.10644800464312236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,1,128,1,float16,float16,0,0.055776000022888184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,1,128,1,float16,fp8,0,0.06173333525657654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,8,128,1,float16,fp8,0,0.14268799622853598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,2,128,1,float16,float16,0,0.10334400335947673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,2,128,1,float16,float16,0,0.058506667613983154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,2,128,1,float16,fp8,0,0.06633066634337108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,4,128,1,float16,float16,0,0.06717866659164429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,4,128,1,float16,fp8,0,0.07753066718578339
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,8,128,1,float16,float16,0,0.07015466690063477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,8,128,1,float16,fp8,0,0.07938666641712189
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,48,128,1,float16,fp8,0,0.056330665946006775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,48,128,1,float16,float16,0,0.06083733340104421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,1,128,1,float16,float16,0,0.027098665634791057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,1,128,1,float16,fp8,0,0.03160533308982849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,2,128,1,float16,float16,0,0.02888533224662145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,2,128,1,float16,fp8,0,0.033285332222779594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,4,128,1,float16,float16,0,0.03316266586383184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,4,128,1,float16,fp8,0,0.036464000741640724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,8,128,1,float16,fp8,0,0.037077332536379494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,48,128,1,float16,float16,0,0.031354665756225586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,48,128,1,float16,fp8,0,0.026335999369621277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,2,128,1,float16,float16,0,0.02070933332045873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,8,128,1,float16,float16,0,0.03500800083080927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,1,128,1,float16,float16,0,0.020810666183630627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,1,128,1,float16,fp8,0,0.024218666056791942
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,2,128,1,float16,fp8,0,0.024080000817775726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,4,128,1,float16,fp8,0,0.025775998830795288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,4,128,1,float16,float16,0,0.022197333474953968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,8,128,1,float16,float16,0,0.02213866760333379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,8,128,1,float16,fp8,0,0.025914666553338368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,48,128,1,float16,float16,0,0.018570666511853535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,48,128,1,float16,fp8,0,0.019386666516462963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,1,128,1,float16,float16,0,0.015066667149464289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,1,128,1,float16,fp8,0,0.01773333301146825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,2,128,1,float16,float16,0,0.015568000574906668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,2,128,1,float16,fp8,0,0.017632000148296356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,4,128,1,float16,float16,0,0.015749332805474598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,4,128,1,float16,fp8,0,0.018325333793958027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,8,128,1,float16,float16,0,0.01603200038274129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,8,128,1,float16,fp8,0,0.018298666924238205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,48,128,1,float16,float16,0,0.014261333892742792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,48,128,1,float16,fp8,0,0.014789332946141561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,1,128,1,float16,float16,0,0.014981333166360855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,2,128,1,float16,fp8,0,0.018090666582187016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,1,128,1,float16,fp8,0,0.01798933371901512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,2,128,1,float16,float16,0,0.01552533358335495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,4,128,1,float16,float16,0,0.01545599972208341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,4,128,1,float16,fp8,0,0.018079999834299088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,8,128,1,float16,float16,0,0.01515199989080429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,48,128,1,float16,float16,0,0.01370666672786077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,8,128,1,float16,fp8,0,0.017583999782800674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,48,128,1,float16,fp8,0,0.014954666296641031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,1,128,1,float16,float16,0,0.01452800010641416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,1,128,1,float16,fp8,0,0.01736533393462499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,2,128,1,float16,float16,0,0.014997333288192749
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,2,128,1,float16,fp8,0,0.01732800031701724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,4,128,1,float16,float16,0,0.014938666174809137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,4,128,1,float16,fp8,0,0.017509333789348602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,8,128,1,float16,float16,0,0.015040000279744467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,8,128,1,float16,fp8,0,0.017535999417304993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,1,128,1,float16,float16,0,0.17074666420618692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,2,128,1,float16,float16,0,0.18725866079330444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,2,128,1,float16,fp8,0,0.20333866278330484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,4,128,1,float16,float16,0,0.22145066658655801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,1,128,1,float16,fp8,0,0.1917919913927714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,4,128,1,float16,fp8,0,0.25332266092300415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,48,128,1,float16,float16,0,0.26505066951115924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,8,128,1,float16,fp8,0,0.29502934217453003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,48,128,1,float16,fp8,0,0.25574400027592975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,1,128,1,float16,float16,0,0.09392533699671428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,1,128,1,float16,fp8,0,0.10922132929166158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,2,128,1,float16,float16,0,0.10192533334096272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,2,128,1,float16,fp8,0,0.11667733391125996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,48,128,1,float16,float16,0,0.13970133662223816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,48,128,1,float16,fp8,0,0.1365386644999186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,8,128,1,float16,fp8,0,0.1625493367513021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,4,128,1,float16,fp8,0,0.1399999956289927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,1,128,1,float16,float16,0,0.05565866827964783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,4,128,1,float16,float16,0,0.11844799915949504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,8,128,1,float16,float16,0,0.2650826573371887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,1,128,1,float16,fp8,0,0.06115200122197469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,8,128,1,float16,float16,0,0.13900799552599588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,2,128,1,float16,fp8,0,0.06648533542950948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,2,128,1,float16,float16,0,0.060032000144322716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,4,128,1,float16,float16,0,0.06858666737874348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,4,128,1,float16,fp8,0,0.0779306689898173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,8,128,1,float16,float16,0,0.07854933540026347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,8,128,1,float16,fp8,0,0.09038399656613667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,48,128,1,float16,fp8,0,0.0751039981842041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,48,128,1,float16,float16,0,0.07740800082683563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,1,128,1,float16,float16,0,0.026933332284291584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,1,128,1,float16,fp8,0,0.03169599920511246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,8,128,1,float16,float16,0,0.03897066662708918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,2,128,1,float16,float16,0,0.029103999336560566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,2,128,1,float16,fp8,0,0.03323200096686681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,4,128,1,float16,float16,0,0.03176533430814743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,4,128,1,float16,fp8,0,0.037104000647862755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,8,128,1,float16,fp8,0,0.042223999897638954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,48,128,1,float16,float16,0,0.0407679999868075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,48,128,1,float16,fp8,0,0.037087999284267426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,1,128,1,float16,float16,0,0.02054399996995926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,1,128,1,float16,fp8,0,0.02422933280467987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,2,128,1,float16,float16,0,0.020794666061798733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,2,128,1,float16,fp8,0,0.02420799930890401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,4,128,1,float16,float16,0,0.022181332111358643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,4,128,1,float16,fp8,0,0.02586666742960612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,8,128,1,float16,float16,0,0.022074667116006214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,8,128,1,float16,fp8,0,0.025920001169045765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,48,128,1,float16,float16,0,0.02239999920129776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,48,128,1,float16,fp8,0,0.022709332406520844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,1,128,1,float16,float16,0,0.015370666980743408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,1,128,1,float16,fp8,0,0.017802666872739792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,2,128,1,float16,float16,0,0.015429332852363586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,2,128,1,float16,fp8,0,0.018021332720915478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,4,128,1,float16,float16,0,0.015583999454975128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,4,128,1,float16,fp8,0,0.01828266680240631
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,8,128,1,float16,float16,0,0.01552533358335495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,8,128,1,float16,fp8,0,0.018021332720915478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,48,128,1,float16,float16,0,0.017445333302021027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,2,128,1,float16,float16,0,0.01523200049996376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,48,128,1,float16,fp8,0,0.018042666216691334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,1,128,1,float16,float16,0,0.015114666273196539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,1,128,1,float16,fp8,0,0.017621333400408428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,2,128,1,float16,fp8,0,0.017605333278576534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,4,128,1,float16,float16,0,0.015013333410024643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,8,128,1,float16,float16,0,0.015178666760524115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,4,128,1,float16,fp8,0,0.017616000026464462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,8,128,1,float16,fp8,0,0.0173333336909612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,48,128,1,float16,float16,0,0.013637332866589228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,48,128,1,float16,fp8,0,0.014789332946141561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,1,128,1,float16,float16,0,0.014858666807413101
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,1,128,1,float16,fp8,0,0.01752000053723653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,2,128,1,float16,float16,0,0.015285332997639975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,2,128,1,float16,fp8,0,0.017397332936525345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,4,128,1,float16,float16,0,0.014970666418472925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,4,128,1,float16,fp8,0,0.01741333305835724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,8,128,1,float16,float16,0,0.01629866659641266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,8,128,1,float16,fp8,0,0.017429333180189133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,48,128,1,float16,float16,0,0.013631999492645264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,48,128,1,float16,fp8,0,0.01481066644191742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,1,128,1,float16,float16,0,0.014218666901191076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,8,128,1,float16,float16,0,0.01421333352724711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,1,128,1,float16,fp8,0,0.016362667083740234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,2,128,1,float16,float16,0,0.014042666802803675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,2,128,1,float16,fp8,0,0.0161013330022494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,4,128,1,float16,float16,0,0.014277332772811254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,4,128,1,float16,fp8,0,0.016336000214020412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,8,128,1,float16,fp8,0,0.016399999459584553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,1,128,1,float16,fp8,0,0.10922666390736897
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,1,128,1,float16,float16,0,0.09353066484133403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,2,128,1,float16,fp8,0,0.11654399832089742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,2,128,1,float16,float16,0,0.10236799716949463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,4,128,1,float16,float16,0,0.13477866848309836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,4,128,1,float16,fp8,0,0.1583626667658488
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,48,128,1,float16,float16,0,0.21724265813827515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,8,128,1,float16,float16,0,0.14126933614412943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,48,128,1,float16,fp8,0,0.2097973426183065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,8,128,1,float16,fp8,0,0.16101866960525513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,1,128,1,float16,float16,0,0.055311997731526695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,1,128,1,float16,fp8,0,0.06261333326498668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,2,128,1,float16,float16,0,0.06026133398214976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,2,128,1,float16,fp8,0,0.06611733138561249
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,4,128,1,float16,fp8,0,0.08779733379681905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,48,128,1,float16,float16,0,0.11414933204650879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,8,128,1,float16,fp8,0,0.08986666798591614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,4,128,1,float16,float16,0,0.07669333120187123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,8,128,1,float16,float16,0,0.07969066500663757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,2,128,1,float16,float16,0,0.02865600089232127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,48,128,1,float16,fp8,0,0.11221333344777425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,1,128,1,float16,float16,0,0.02701866626739502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,1,128,1,float16,fp8,0,0.031343999008337654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,4,128,1,float16,float16,0,0.03585066646337509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,2,128,1,float16,fp8,0,0.03313066562016805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,4,128,1,float16,fp8,0,0.04178133110205332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,8,128,1,float16,float16,0,0.04001066585381826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,8,128,1,float16,fp8,0,0.04334400097529093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,48,128,1,float16,float16,0,0.060085331400235496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,48,128,1,float16,fp8,0,0.05406933526198069
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,1,128,1,float16,float16,0,0.02059200033545494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,1,128,1,float16,fp8,0,0.02426133304834366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,2,128,1,float16,float16,0,0.020618667205174763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,2,128,1,float16,fp8,0,0.02403733382622401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,4,128,1,float16,float16,0,0.0222080002228419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,4,128,1,float16,fp8,0,0.02584533393383026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,8,128,1,float16,float16,0,0.022101332743962605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,1,128,1,float16,fp8,0,0.017797333498795826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,8,128,1,float16,fp8,0,0.02571733295917511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,48,128,1,float16,float16,0,0.0312266672650973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,48,128,1,float16,fp8,0,0.031178665657838184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,1,128,1,float16,float16,0,0.015354666858911514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,2,128,1,float16,float16,0,0.01635733370979627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,2,128,1,float16,fp8,0,0.018426666657129925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,4,128,1,float16,float16,0,0.01575999955336253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,8,128,1,float16,float16,0,0.01553600033124288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,4,128,1,float16,fp8,0,0.01807466646035512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,8,128,1,float16,fp8,0,0.017898666361967724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,48,128,1,float16,float16,0,0.02075200031201045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,48,128,1,float16,fp8,0,0.02143999934196472
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,1,128,1,float16,float16,0,0.015119999647140503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,4,128,1,float16,fp8,0,0.01729600007335345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,1,128,1,float16,fp8,0,0.017893332988023758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,2,128,1,float16,float16,0,0.015295999745527903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,8,128,1,float16,float16,0,0.014725333700577417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,2,128,1,float16,fp8,0,0.017514667163292568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,4,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,8,128,1,float16,fp8,0,0.017632000148296356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,48,128,1,float16,float16,0,0.01687466725707054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,48,128,1,float16,fp8,0,0.017978666971127193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,1,128,1,float16,float16,0,0.015002666662136713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,1,128,1,float16,fp8,0,0.01672533278663953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,2,128,1,float16,float16,0,0.015034666905800501
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,2,128,1,float16,fp8,0,0.017349333812793095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,8,128,1,float16,fp8,0,0.017423999806245167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,4,128,1,float16,float16,0,0.014762666076421738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,48,128,1,float16,fp8,0,0.01451733335852623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,4,128,1,float16,fp8,0,0.01724799970785777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,8,128,1,float16,float16,0,0.014645333091417948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,48,128,1,float16,float16,0,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,1,128,1,float16,float16,0,0.014085333794355392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,1,128,1,float16,fp8,0,0.016255999604860943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,2,128,1,float16,float16,0,0.01421333352724711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,2,128,1,float16,fp8,0,0.01618133361140887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,4,128,1,float16,float16,0,0.014842666685581207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,4,128,1,float16,fp8,0,0.016389333953460056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,8,128,1,float16,float16,0,0.014202666779359182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,8,128,1,float16,fp8,0,0.0162773331006368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,48,128,1,float16,fp8,0,0.014495999862750372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,48,128,1,float16,float16,0,0.01349866638580958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,1,128,1,float16,float16,0,0.01394133393963178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,1,128,1,float16,fp8,0,0.016085332880417507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,2,128,1,float16,float16,0,0.013872000078360239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,2,128,1,float16,fp8,0,0.01617066686352094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,4,128,1,float16,float16,0,0.01414399966597557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,4,128,1,float16,fp8,0,0.016048000504573185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,8,128,1,float16,float16,0,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,8,128,1,float16,fp8,0,0.016778666526079178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,1,128,1,float16,fp8,0,34.034767150878906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,2,128,1,float16,fp8,0,34.19246927897135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,4,128,1,float16,fp8,0,35.00013224283854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,1,128,1,float16,float16,0,37.722033182779946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,8,128,1,float16,fp8,0,35.44705708821615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,2,128,1,float16,float16,0,38.180798848470054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,4,128,1,float16,float16,0,38.320159912109375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,8,128,1,float16,float16,0,38.07704416910807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,40,128,1,float16,fp8,0,18.344095865885418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,1,128,1,float16,fp8,0,17.092325846354168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,40,128,1,float16,float16,0,20.836437225341797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,1,128,1,float16,float16,0,19.22664515177409
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,2,128,1,float16,fp8,0,17.180421193440754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,2,128,1,float16,float16,0,20.081429799397785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,4,128,1,float16,fp8,0,17.515684763590496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,4,128,1,float16,float16,0,19.147664388020832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,40,128,1,float16,fp8,0,9.236245473225912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,40,128,1,float16,float16,0,10.39030392964681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,1,128,1,float16,float16,0,10.391567866007486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,1,128,1,float16,fp8,0,8.766255696614584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,2,128,1,float16,float16,0,9.52351442972819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,2,128,1,float16,fp8,0,8.644159952799479
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,8,128,1,float16,fp8,0,18.161258697509766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,40,128,1,float16,fp8,0,4.743498802185059
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,40,128,1,float16,float16,0,5.219615936279297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,8,128,1,float16,float16,0,20.537588755289715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,4,128,1,float16,float16,0,11.298144022623697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,4,128,1,float16,fp8,0,9.506687800089518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,8,128,1,float16,fp8,0,10.206245422363281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,8,128,1,float16,float16,0,9.757189432779947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,1,128,1,float16,float16,0,5.168778737386067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,1,128,1,float16,fp8,0,4.754634539286296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,2,128,1,float16,float16,0,5.403578440348308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,2,128,1,float16,fp8,0,5.041232109069824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,4,128,1,float16,fp8,0,4.5176693598429365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,8,128,1,float16,float16,0,5.173424084981282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,4,128,1,float16,float16,0,5.44874636332194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,8,128,1,float16,fp8,0,4.925306638081868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,1,128,1,float16,fp8,0,20.085962931315105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,2,128,1,float16,fp8,0,19.76367950439453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,1,128,1,float16,float16,0,22.20140329996745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,2,128,1,float16,float16,0,21.70361582438151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,4,128,1,float16,fp8,0,20.399220784505207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,4,128,1,float16,float16,0,22.49786631266276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,8,128,1,float16,fp8,0,20.327152252197266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,8,128,1,float16,float16,0,22.895273844401043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,40,128,1,float16,float16,0,11.897327423095703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,40,128,1,float16,fp8,0,10.774954477945963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,1,128,1,float16,fp8,0,9.808202743530273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,1,128,1,float16,float16,0,10.922698974609375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,2,128,1,float16,fp8,0,9.871322631835938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,2,128,1,float16,float16,0,11.855354309082031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,4,128,1,float16,fp8,0,10.174186706542969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,4,128,1,float16,float16,0,11.501936594645182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,40,128,1,float16,float16,0,5.600101470947266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,40,128,1,float16,fp8,0,5.532085418701172
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,1,128,1,float16,float16,0,4.9645334879557295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,1,128,1,float16,fp8,0,5.019722620646159
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,2,128,1,float16,float16,0,5.089418729146321
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,8,128,1,float16,fp8,0,10.163818359375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,2,128,1,float16,fp8,0,5.008485476175944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,8,128,1,float16,float16,0,11.369589487711588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,4,128,1,float16,float16,0,5.558149337768555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,4,128,1,float16,fp8,0,5.136629422505696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,40,128,1,float16,float16,0,2.881248156229655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,40,128,1,float16,fp8,0,2.74560546875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,1,128,1,float16,float16,0,2.5657547314961753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,8,128,1,float16,float16,0,5.564042409261067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,8,128,1,float16,fp8,0,5.158912022908528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,1,128,1,float16,fp8,0,2.8045600255330405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,2,128,1,float16,float16,0,2.6380160649617515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,2,128,1,float16,fp8,0,2.7474400202433267
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,4,128,1,float16,float16,0,2.594266732533773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,4,128,1,float16,fp8,0,2.752394676208496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,8,128,1,float16,float16,0,2.6674559911092124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,8,128,1,float16,fp8,0,2.6305813789367676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,1,128,1,float16,fp8,0,13.746426900227865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,1,128,1,float16,float16,0,15.496037801106771
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,2,128,1,float16,fp8,0,13.94909922281901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,2,128,1,float16,float16,0,15.656341552734375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,4,128,1,float16,fp8,0,14.383413950602213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,4,128,1,float16,float16,0,16.1234614054362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,8,128,1,float16,float16,0,16.533184051513672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,40,128,1,float16,fp8,0,7.725946426391602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,1,128,1,float16,float16,0,7.7697493235270185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,1,128,1,float16,fp8,0,7.137653350830078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,40,128,1,float16,float16,0,8.318000157674154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,2,128,1,float16,float16,0,7.912464141845703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,2,128,1,float16,fp8,0,6.98031489054362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,4,128,1,float16,float16,0,8.173253377278646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,8,128,1,float16,fp8,0,14.539562225341797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,1,128,1,float16,float16,0,3.5899734497070312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,40,128,1,float16,float16,0,3.96015993754069
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,40,128,1,float16,fp8,0,4.274997393290202
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,4,128,1,float16,fp8,0,7.22972297668457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,8,128,1,float16,fp8,0,7.233450571695964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,1,128,1,float16,fp8,0,4.362346649169922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,8,128,1,float16,float16,0,7.939706802368164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,2,128,1,float16,float16,0,3.720442771911621
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,2,128,1,float16,fp8,0,3.5433600743611655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,4,128,1,float16,float16,0,3.6913013458251953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,4,128,1,float16,fp8,0,3.845786730448405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,8,128,1,float16,float16,0,3.742351849873861
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,40,128,1,float16,float16,0,2.037871996561686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,40,128,1,float16,fp8,0,2.0629493395487466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,1,128,1,float16,float16,0,1.8582560221354167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,8,128,1,float16,fp8,0,3.6674880981445312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,2,128,1,float16,fp8,0,1.8477546374003093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,2,128,1,float16,float16,0,1.8261760075887044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,4,128,1,float16,float16,0,1.863466739654541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,4,128,1,float16,fp8,0,1.9096266428629558
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,8,128,1,float16,float16,0,1.856767972310384
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,1,128,1,float16,fp8,0,1.8647573788960774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,8,128,1,float16,fp8,0,2.016239960988363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,1,128,1,float16,float16,0,20.441925048828125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,1,128,1,float16,fp8,0,17.951269785563152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,2,128,1,float16,fp8,0,18.105728149414062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,2,128,1,float16,float16,0,20.09173838297526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,4,128,1,float16,fp8,0,18.892186482747395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,8,128,1,float16,fp8,0,18.848005930582683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,4,128,1,float16,float16,0,20.999797821044922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,8,128,1,float16,float16,0,20.70369593302409
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,40,128,1,float16,fp8,0,10.278085072835287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,40,128,1,float16,float16,0,10.981221516927084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,1,128,1,float16,float16,0,10.61350949605306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,1,128,1,float16,fp8,0,9.01359494527181
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,2,128,1,float16,float16,0,10.540751775105795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,2,128,1,float16,fp8,0,9.06606419881185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,4,128,1,float16,float16,0,10.623578389485678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,4,128,1,float16,fp8,0,9.508522669474283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,40,128,1,float16,float16,0,5.342490514119466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,40,128,1,float16,fp8,0,5.1341813405354815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,1,128,1,float16,float16,0,4.756287892659505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,1,128,1,float16,fp8,0,4.503674825032552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,2,128,1,float16,float16,0,4.893088022867839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,2,128,1,float16,fp8,0,4.5556745529174805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,8,128,1,float16,float16,0,10.496394475301107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,8,128,1,float16,fp8,0,9.617045084635416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,4,128,1,float16,fp8,0,4.732063929239909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,4,128,1,float16,float16,0,4.945333480834961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,40,128,1,float16,float16,0,2.690394719441732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,8,128,1,float16,fp8,0,4.768992106119792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,8,128,1,float16,float16,0,5.018416086832683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,40,128,1,float16,fp8,0,2.6249759991963706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,1,128,1,float16,float16,0,2.3101654052734375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,1,128,1,float16,fp8,0,2.3080639839172363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,2,128,1,float16,float16,0,2.333183924357096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,4,128,1,float16,float16,0,2.527130603790283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,4,128,1,float16,fp8,0,2.4083147048950195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,2,128,1,float16,fp8,0,2.571301301320394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,40,128,1,float16,float16,0,1.3327306906382244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,8,128,1,float16,float16,0,2.4883519808451333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,40,128,1,float16,fp8,0,1.356378714243571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,8,128,1,float16,fp8,0,2.4317332903544107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,1,128,1,float16,float16,0,1.2223626772562664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,1,128,1,float16,fp8,0,1.275861342748006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,2,128,1,float16,float16,0,1.1832640171051025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,2,128,1,float16,fp8,0,1.2964320182800293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,4,128,1,float16,float16,0,1.2258986632029216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,4,128,1,float16,fp8,0,1.3873279889424641
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,8,128,1,float16,float16,0,1.2459200223286946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,8,128,1,float16,fp8,0,1.2578826745351155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,1,128,1,float16,fp8,0,10.489456176757812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,2,128,1,float16,fp8,0,10.579605102539062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,2,128,1,float16,float16,0,11.872720082600912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,1,128,1,float16,float16,0,11.438613891601562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,4,128,1,float16,fp8,0,11.193215688069662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,4,128,1,float16,float16,0,12.379892985026041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,8,128,1,float16,fp8,0,11.137626647949219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,8,128,1,float16,float16,0,12.383771260579428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,40,128,1,float16,float16,0,6.434384028116862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,1,128,1,float16,float16,0,5.422026952107747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,1,128,1,float16,fp8,0,5.23910395304362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,40,128,1,float16,fp8,0,6.3329118092854815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,2,128,1,float16,float16,0,5.521680196126302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,2,128,1,float16,fp8,0,5.297290802001953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,4,128,1,float16,float16,0,5.9995466868082685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,40,128,1,float16,float16,0,3.0641441345214844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,1,128,1,float16,float16,0,2.654815991719564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,40,128,1,float16,fp8,0,3.4871091842651367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,1,128,1,float16,fp8,0,2.832373301188151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,4,128,1,float16,fp8,0,5.632975896199544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,8,128,1,float16,float16,0,6.122581481933594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,8,128,1,float16,fp8,0,5.597109476725261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,2,128,1,float16,float16,0,2.6930294036865234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,4,128,1,float16,float16,0,2.792480150858561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,2,128,1,float16,fp8,0,3.042917251586914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,40,128,1,float16,float16,0,1.557088057200114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,8,128,1,float16,float16,0,2.8811041514078775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,4,128,1,float16,fp8,0,2.853562672932943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,40,128,1,float16,fp8,0,1.7430453300476074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,8,128,1,float16,fp8,0,3.1892372767130532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,1,128,1,float16,float16,0,1.3389387130737305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,1,128,1,float16,fp8,0,1.4152746200561523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,2,128,1,float16,float16,0,1.3541919390360515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,2,128,1,float16,fp8,0,1.5316640535990398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,4,128,1,float16,float16,0,1.4274293581644695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,8,128,1,float16,float16,0,1.4224106470743816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,40,128,1,float16,float16,0,0.8108747005462646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,8,128,1,float16,fp8,0,1.4659786224365234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,40,128,1,float16,fp8,0,0.8419840335845947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,1,128,1,float16,float16,0,0.7012373606363932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,4,128,1,float16,fp8,0,1.4872959454854329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,2,128,1,float16,fp8,0,0.7470880349477133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,1,128,1,float16,fp8,0,0.7117599646250407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,2,128,1,float16,float16,0,0.7070826689402262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,4,128,1,float16,fp8,0,0.7582773367563883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,4,128,1,float16,float16,0,0.7319839795430502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,8,128,1,float16,float16,0,0.7399199803670248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,8,128,1,float16,fp8,0,0.7885973453521729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,1,128,1,float16,fp8,0,10.085626602172852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,2,128,1,float16,fp8,0,10.148037592569986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,2,128,1,float16,float16,0,11.517781575520834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,4,128,1,float16,float16,0,12.056512196858725
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,4,128,1,float16,fp8,0,11.030885060628256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,8,128,1,float16,float16,0,11.586128234863281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,1,128,1,float16,float16,0,10.824026743570963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,8,128,1,float16,fp8,0,11.06268310546875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,1,128,1,float16,float16,0,5.311882654825847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,40,128,1,float16,float16,0,6.233845392862956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,40,128,1,float16,fp8,0,6.235418955485026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,2,128,1,float16,float16,0,5.328405380249023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,1,128,1,float16,fp8,0,5.170463879903157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,2,128,1,float16,fp8,0,5.07971731821696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,4,128,1,float16,float16,0,5.453301111857097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,4,128,1,float16,fp8,0,5.474645614624023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,40,128,1,float16,float16,0,3.070624033610026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,1,128,1,float16,float16,0,2.4588905970255532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,1,128,1,float16,fp8,0,2.5001279513041177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,40,128,1,float16,fp8,0,3.4034932454427085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,2,128,1,float16,fp8,0,2.527946631113688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,2,128,1,float16,float16,0,2.5485493342081704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,8,128,1,float16,float16,0,5.771738688151042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,8,128,1,float16,fp8,0,5.631365458170573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,4,128,1,float16,float16,0,2.7334025700887046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,40,128,1,float16,float16,0,1.5305867195129395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,1,128,1,float16,float16,0,1.2960320313771565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,4,128,1,float16,fp8,0,2.90664005279541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,8,128,1,float16,float16,0,2.7097705205281577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,40,128,1,float16,fp8,0,1.7084213892618816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,8,128,1,float16,fp8,0,2.8868319193522134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,1,128,1,float16,fp8,0,1.2825013001759846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,2,128,1,float16,fp8,0,1.3177440166473389
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,2,128,1,float16,float16,0,1.3055360317230225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,4,128,1,float16,float16,0,1.4072640736897786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,8,128,1,float16,float16,0,1.4871306419372559
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,4,128,1,float16,fp8,0,1.4501973787943523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,40,128,1,float16,float16,0,0.7863626480102539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,1,128,1,float16,float16,0,0.6356853246688843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,8,128,1,float16,fp8,0,1.4891467094421387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,40,128,1,float16,fp8,0,0.831653356552124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,1,128,1,float16,fp8,0,0.6773973306020101
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,2,128,1,float16,float16,0,0.6733280022939047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,2,128,1,float16,fp8,0,0.7423413594563802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,40,128,1,float16,float16,0,0.4131253163019816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,4,128,1,float16,float16,0,0.6964159806569418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,8,128,1,float16,float16,0,0.6986453533172607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,4,128,1,float16,fp8,0,0.7305493354797363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,8,128,1,float16,fp8,0,0.7428106466929117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,40,128,1,float16,fp8,0,0.45208001136779785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,1,128,1,float16,float16,0,0.3424533208211263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,1,128,1,float16,fp8,0,0.37245333194732666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,8,128,1,float16,float16,0,0.3717813491821289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,2,128,1,float16,float16,0,0.3421226739883423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,2,128,1,float16,fp8,0,0.36983998616536456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,4,128,1,float16,float16,0,0.3656160036722819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,4,128,1,float16,fp8,0,0.3893440167109172
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,8,128,1,float16,fp8,0,0.40218667189280194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,1,128,1,float16,float16,0,6.388282775878906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,2,128,1,float16,float16,0,6.469184239705403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,1,128,1,float16,fp8,0,5.954213460286458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,2,128,1,float16,fp8,0,6.1868851979573565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,4,128,1,float16,float16,0,6.783152262369792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,8,128,1,float16,float16,0,6.609082539876302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,4,128,1,float16,fp8,0,6.686992009480794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,40,128,1,float16,float16,0,3.80399481455485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,1,128,1,float16,float16,0,2.9138240814208984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,1,128,1,float16,fp8,0,3.265514691670736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,2,128,1,float16,float16,0,3.1275466283162436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,40,128,1,float16,fp8,0,3.9527146021525064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,2,128,1,float16,fp8,0,3.034447987874349
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,8,128,1,float16,fp8,0,6.953519821166992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,1,128,1,float16,float16,0,1.4866879781087239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,4,128,1,float16,float16,0,3.267258644104004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,8,128,1,float16,float16,0,3.2167094548543296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,40,128,1,float16,fp8,0,1.986240069071452
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,4,128,1,float16,fp8,0,3.431056022644043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,40,128,1,float16,float16,0,1.9162507057189941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,8,128,1,float16,fp8,0,3.38916810353597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,1,128,1,float16,fp8,0,1.520949363708496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,2,128,1,float16,float16,0,1.5340266227722168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,2,128,1,float16,fp8,0,1.5783467292785645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,4,128,1,float16,float16,0,1.5974292755126953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,40,128,1,float16,float16,0,0.9722293217976888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,4,128,1,float16,fp8,0,1.7376160621643066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,40,128,1,float16,fp8,0,1.027786652247111
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,8,128,1,float16,float16,0,1.6431199709574382
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,1,128,1,float16,float16,0,0.7552533149719238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,2,128,1,float16,float16,0,0.7622826894124349
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,1,128,1,float16,fp8,0,0.799845298131307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,4,128,1,float16,fp8,0,0.894981304804484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,4,128,1,float16,float16,0,0.8263733386993408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,40,128,1,float16,float16,0,0.4929973284403483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,8,128,1,float16,float16,0,0.8359359900156657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,2,128,1,float16,fp8,0,0.8162026405334473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,40,128,1,float16,fp8,0,0.5463519891103109
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,8,128,1,float16,fp8,0,0.9176533222198486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,8,128,1,float16,fp8,0,1.7339413960774739
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,1,128,1,float16,float16,0,0.3960213263829549
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,1,128,1,float16,fp8,0,0.41971735159556073
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,2,128,1,float16,float16,0,0.3932266632715861
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,2,128,1,float16,fp8,0,0.4185440142949422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,4,128,1,float16,float16,0,0.43195199966430664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,4,128,1,float16,fp8,0,0.46619733174641925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,8,128,1,float16,float16,0,0.43701334794362384
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,40,128,1,float16,float16,0,0.2763520081837972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,8,128,1,float16,fp8,0,0.4680639902750651
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,40,128,1,float16,fp8,0,0.2983520030975342
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,1,128,1,float16,float16,0,0.2075093388557434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,1,128,1,float16,fp8,0,0.22687999407450357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,2,128,1,float16,float16,0,0.22116265694300333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,2,128,1,float16,fp8,0,0.23405865828196207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,4,128,1,float16,float16,0,0.23187732696533203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,4,128,1,float16,fp8,0,0.2537173430124919
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,8,128,1,float16,fp8,0,0.259935994942983
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,8,128,1,float16,float16,0,0.23418132464090982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,1,128,1,float16,fp8,0,5.911231994628906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,2,128,1,float16,float16,0,5.913749059041341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,4,128,1,float16,float16,0,6.486586888631185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,2,128,1,float16,fp8,0,6.086869557698567
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,1,128,1,float16,float16,0,5.792954762776692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,4,128,1,float16,fp8,0,6.901546478271484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,8,128,1,float16,float16,0,6.869530359903972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,8,128,1,float16,fp8,0,6.911226908365886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,1,128,1,float16,float16,0,2.83022403717041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,2,128,1,float16,float16,0,2.9826666514078775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,40,128,1,float16,float16,0,3.982639948527018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,1,128,1,float16,fp8,0,2.970464070638021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,2,128,1,float16,fp8,0,3.0599466959635415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,4,128,1,float16,float16,0,3.3264853159586587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,40,128,1,float16,fp8,0,4.449642817179362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,4,128,1,float16,fp8,0,3.468384106953939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,40,128,1,float16,float16,0,2.0073493321736655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,8,128,1,float16,float16,0,3.2740586598714194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,1,128,1,float16,float16,0,1.4362773895263672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,2,128,1,float16,float16,0,1.483290672302246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,1,128,1,float16,fp8,0,1.5601013501485188
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,40,128,1,float16,fp8,0,2.1298933029174805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,2,128,1,float16,fp8,0,1.5468106269836426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,8,128,1,float16,fp8,0,3.4796320597330728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,4,128,1,float16,float16,0,1.6393334070841472
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,8,128,1,float16,float16,0,1.675279935201009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,4,128,1,float16,fp8,0,1.8110079765319824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,40,128,1,float16,float16,0,1.018170674641927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,8,128,1,float16,fp8,0,1.7839412689208984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,40,128,1,float16,fp8,0,1.0816853046417236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,1,128,1,float16,float16,0,0.7133599917093912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,1,128,1,float16,fp8,0,0.7706773281097412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,2,128,1,float16,float16,0,0.7640533447265625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,2,128,1,float16,fp8,0,0.8104586601257324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,4,128,1,float16,float16,0,0.8307680288950602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,8,128,1,float16,float16,0,0.8477173646291097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,4,128,1,float16,fp8,0,0.9139946301778158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,40,128,1,float16,float16,0,0.5130506753921509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,8,128,1,float16,fp8,0,0.9043786525726318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,40,128,1,float16,fp8,0,0.5636853377024332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,1,128,1,float16,float16,0,0.37969064712524414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,1,128,1,float16,fp8,0,0.41070401668548584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,2,128,1,float16,float16,0,0.39561601479848224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,2,128,1,float16,fp8,0,0.419594685236613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,4,128,1,float16,float16,0,0.4330666859944661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,4,128,1,float16,fp8,0,0.46114134788513184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,8,128,1,float16,float16,0,0.4313013156255086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,40,128,1,float16,float16,0,0.28175999720891315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,8,128,1,float16,fp8,0,0.48390400409698486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,40,128,1,float16,fp8,0,0.30606400966644287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,1,128,1,float16,float16,0,0.20511466264724731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,1,128,1,float16,fp8,0,0.2225333253542582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,2,128,1,float16,float16,0,0.2088800072669983
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,2,128,1,float16,fp8,0,0.22376000881195068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,4,128,1,float16,float16,0,0.23369600375493368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,4,128,1,float16,fp8,0,0.2537600000699361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,8,128,1,float16,float16,0,0.2366559902826945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,8,128,1,float16,fp8,0,0.2594613234202067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,40,128,1,float16,float16,0,0.1551413337389628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,40,128,1,float16,fp8,0,0.1678453286488851
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,1,128,1,float16,float16,0,0.11056533455848694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,1,128,1,float16,fp8,0,0.12024533748626709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,2,128,1,float16,float16,0,0.12060266733169556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,2,128,1,float16,fp8,0,0.12891733646392822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,4,128,1,float16,float16,0,0.12945066889127096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,8,128,1,float16,float16,0,0.13133333126703897
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,4,128,1,float16,fp8,0,0.1421280006567637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,8,128,1,float16,fp8,0,0.1444960037867228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,1,128,1,float16,float16,0,3.475338617960612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,1,128,1,float16,fp8,0,3.7779998779296875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,4,128,1,float16,float16,0,4.057658513387044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,2,128,1,float16,fp8,0,3.828895886739095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,8,128,1,float16,float16,0,4.119615872701009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,4,128,1,float16,fp8,0,4.48964277903239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,2,128,1,float16,float16,0,3.6049172083536782
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,1,128,1,float16,float16,0,1.7862666447957356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,40,128,1,float16,fp8,0,2.8278239568074546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,1,128,1,float16,fp8,0,1.8752373059590657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,2,128,1,float16,float16,0,1.852453390757243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,2,128,1,float16,fp8,0,1.939066727956136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,8,128,1,float16,fp8,0,4.7578080495198565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,4,128,1,float16,float16,0,2.112208048502604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,4,128,1,float16,fp8,0,2.3130027453104653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,8,128,1,float16,float16,0,2.0702293713887534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,1,128,1,float16,float16,0,0.8822186787923177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,40,128,1,float16,float16,0,1.3088266849517822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,8,128,1,float16,fp8,0,2.250773270924886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,40,128,1,float16,fp8,0,1.4324480692545574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,1,128,1,float16,fp8,0,0.9693813323974609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,2,128,1,float16,float16,0,0.9382987022399902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,4,128,1,float16,float16,0,1.040064016977946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,2,128,1,float16,fp8,0,0.9818507035573324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,8,128,1,float16,float16,0,1.0443306763966878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,40,128,1,float16,float16,0,2.569215933481852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,1,128,1,float16,float16,0,0.4516106843948364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,40,128,1,float16,float16,0,0.6671520074208578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,1,128,1,float16,fp8,0,0.49517865975697833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,2,128,1,float16,float16,0,0.4733653465906779
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,4,128,1,float16,fp8,0,1.1619306405385335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,40,128,1,float16,fp8,0,0.7469973564147949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,8,128,1,float16,fp8,0,1.1544640064239502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,4,128,1,float16,float16,0,0.53875199953715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,4,128,1,float16,fp8,0,0.5974773168563843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,8,128,1,float16,float16,0,0.5427039861679077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,8,128,1,float16,fp8,0,0.6041173140207926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,2,128,1,float16,fp8,0,0.5179786682128906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,40,128,1,float16,float16,0,0.35277867317199707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,40,128,1,float16,fp8,0,0.3867786725362142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,1,128,1,float16,float16,0,0.23973333835601807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,1,128,1,float16,fp8,0,0.26708799600601196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,2,128,1,float16,float16,0,0.25626667340596515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,2,128,1,float16,fp8,0,0.2738719979921977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,4,128,1,float16,float16,0,0.2834399938583374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,4,128,1,float16,fp8,0,0.3146773378054301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,8,128,1,float16,float16,0,0.28837867577870685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,8,128,1,float16,fp8,0,0.3156320055325826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,40,128,1,float16,float16,0,0.1872373421986898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,40,128,1,float16,fp8,0,0.21369065841039023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,1,128,1,float16,float16,0,0.1344533363978068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,1,128,1,float16,fp8,0,0.14710399508476257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,2,128,1,float16,float16,0,0.13985066612561545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,2,128,1,float16,fp8,0,0.15363199512163797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,4,128,1,float16,float16,0,0.15282133221626282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,4,128,1,float16,fp8,0,0.17044266064961752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,8,128,1,float16,float16,0,0.15591466426849365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,8,128,1,float16,fp8,0,0.17618133624394736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,40,128,1,float16,float16,0,0.10750400026639302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,40,128,1,float16,fp8,0,0.12212799986203511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,1,128,1,float16,float16,0,0.07262933254241943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,1,128,1,float16,fp8,0,0.08213333288828532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,2,128,1,float16,float16,0,0.0765066643555959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,2,128,1,float16,fp8,0,0.0841439962387085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,4,128,1,float16,float16,0,0.0846666693687439
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,4,128,1,float16,fp8,0,0.0916426678498586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,8,128,1,float16,float16,0,0.08930133779843648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,8,128,1,float16,fp8,0,0.09889599680900574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,1,128,1,float16,float16,0,3.55078919728597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,8,128,1,float16,float16,0,4.414165178934733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,2,128,1,float16,float16,0,3.845813433329264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,2,128,1,float16,fp8,0,4.117322603861491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,8,128,1,float16,fp8,0,4.947344144185384
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,1,128,1,float16,fp8,0,3.9985227584838867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,4,128,1,float16,float16,0,4.390384038289388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,40,128,1,float16,float16,0,2.9310986200968423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,4,128,1,float16,fp8,0,4.9819841384887695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,1,128,1,float16,float16,0,1.7829492886861165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,1,128,1,float16,fp8,0,1.9920427004496257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,2,128,1,float16,float16,0,1.9287412961324055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,40,128,1,float16,fp8,0,3.347397486368815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,2,128,1,float16,fp8,0,2.0717493693033853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,4,128,1,float16,fp8,0,2.4843467076619468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,8,128,1,float16,float16,0,2.208090623219808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,4,128,1,float16,float16,0,2.2528799374898276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,40,128,1,float16,float16,0,1.4587732950846355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,8,128,1,float16,fp8,0,2.493312040964762
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,1,128,1,float16,float16,0,0.9107360045115153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,40,128,1,float16,fp8,0,1.6773494084676106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,1,128,1,float16,fp8,0,1.0101919968922932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,2,128,1,float16,float16,0,0.9726666609446207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,2,128,1,float16,fp8,0,1.0704320271809895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,8,128,1,float16,float16,0,1.1400907039642334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,40,128,1,float16,float16,0,0.7479466597239176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,8,128,1,float16,fp8,0,1.2941439946492512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,4,128,1,float16,fp8,0,1.253391981124878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,4,128,1,float16,float16,0,1.1162506739298503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,40,128,1,float16,fp8,0,0.8656266530354818
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,1,128,1,float16,float16,0,0.4663360118865967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,1,128,1,float16,fp8,0,0.5249119997024536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,2,128,1,float16,float16,0,0.5021546681722006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,2,128,1,float16,fp8,0,0.5415093501408895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,4,128,1,float16,float16,0,0.5686986843744913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,8,128,1,float16,float16,0,0.5886720021565756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,4,128,1,float16,fp8,0,0.663045326868693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,40,128,1,float16,float16,0,0.38972798983256024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,1,128,1,float16,float16,0,0.24420799811681113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,8,128,1,float16,fp8,0,0.6640906731287638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,1,128,1,float16,fp8,0,0.2739040056864421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,2,128,1,float16,float16,0,0.2701546748479207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,4,128,1,float16,fp8,0,0.341045339902242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,2,128,1,float16,fp8,0,0.28442132472991943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,4,128,1,float16,float16,0,0.308405339717865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,8,128,1,float16,float16,0,0.3024746576944987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,8,128,1,float16,fp8,0,0.34603198369344074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,40,128,1,float16,fp8,0,0.4532906611760457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,40,128,1,float16,float16,0,0.20696532726287842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,40,128,1,float16,fp8,0,0.23903467257817587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,1,128,1,float16,float16,0,0.13493333260218301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,1,128,1,float16,fp8,0,0.1504639983177185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,2,128,1,float16,fp8,0,0.15956266721089682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,2,128,1,float16,float16,0,0.1470133364200592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,4,128,1,float16,float16,0,0.16240533192952475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,8,128,1,float16,float16,0,0.16497066617012024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,8,128,1,float16,fp8,0,0.1892426609992981
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,4,128,1,float16,fp8,0,0.1848426659901937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,40,128,1,float16,fp8,0,0.13436800241470337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,40,128,1,float16,float16,0,0.11550399661064148
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,1,128,1,float16,float16,0,0.07809066772460938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,4,128,1,float16,fp8,0,0.10329066713651021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,1,128,1,float16,fp8,0,0.0864533285299937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,2,128,1,float16,float16,0,0.08067200084527333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,2,128,1,float16,fp8,0,0.09065600236256917
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,4,128,1,float16,float16,0,0.09141866366068523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,8,128,1,float16,float16,0,0.09230400125185649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,40,128,1,float16,float16,0,0.06866133213043213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,8,128,1,float16,fp8,0,0.1072106659412384
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,40,128,1,float16,fp8,0,0.07997333506743114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,1,128,1,float16,float16,0,0.04526400069395701
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,1,128,1,float16,fp8,0,0.05100800096988678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,2,128,1,float16,float16,0,0.04701866706212362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,2,128,1,float16,fp8,0,0.05438933273156484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,4,128,1,float16,float16,0,0.04986133178075155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,4,128,1,float16,fp8,0,0.05611733098824819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,8,128,1,float16,float16,0,0.05179733534653982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,8,128,1,float16,fp8,0,0.06016000111897787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,1,128,1,float16,float16,0,2.6526293754577637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,1,128,1,float16,fp8,0,3.022810618082682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,2,128,1,float16,float16,0,2.8991467157999673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,2,128,1,float16,fp8,0,3.1715307235717773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,4,128,1,float16,float16,0,3.4504321416219077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,1,128,1,float16,float16,0,1.32805331548055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,40,128,1,float16,float16,0,2.4441280364990234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,4,128,1,float16,fp8,0,4.0480852127075195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,40,128,1,float16,fp8,0,2.9723520278930664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,8,128,1,float16,float16,0,3.5358187357584634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,8,128,1,float16,fp8,0,4.072335879007976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,1,128,1,float16,fp8,0,1.521898587544759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,2,128,1,float16,fp8,0,1.5970239639282227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,2,128,1,float16,float16,0,1.469007968902588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,4,128,1,float16,float16,0,1.7438027064005535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,4,128,1,float16,fp8,0,2.037855943044027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,1,128,1,float16,float16,0,0.6766613324483236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,40,128,1,float16,fp8,0,1.4970614115397136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,8,128,1,float16,float16,0,1.7479573885599773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,1,128,1,float16,fp8,0,0.7764373620351156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,8,128,1,float16,fp8,0,2.0633440017700195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,4,128,1,float16,float16,0,0.8778666655222574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,2,128,1,float16,fp8,0,0.8143306573232015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,2,128,1,float16,float16,0,0.7531946500142416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,8,128,1,float16,fp8,0,1.08188263575236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,4,128,1,float16,fp8,0,1.0343146324157715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,8,128,1,float16,float16,0,0.8973759810129801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,40,128,1,float16,float16,0,1.231007973353068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,40,128,1,float16,fp8,0,0.7624906698862711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,1,128,1,float16,float16,0,0.3487093448638916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,40,128,1,float16,float16,0,0.6236746708552042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,2,128,1,float16,float16,0,0.3898933331171672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,1,128,1,float16,fp8,0,0.4001386562983195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,4,128,1,float16,fp8,0,0.5282026529312134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,4,128,1,float16,float16,0,0.4565173387527466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,2,128,1,float16,fp8,0,0.41945068041483563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,8,128,1,float16,float16,0,0.4628373384475708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,40,128,1,float16,float16,0,0.3264586726824443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,40,128,1,float16,fp8,0,0.39601067701975506
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,1,128,1,float16,float16,0,0.18394132455190024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,8,128,1,float16,fp8,0,0.5544533332188925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,1,128,1,float16,fp8,0,0.21144533157348633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,2,128,1,float16,float16,0,0.20254399379094443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,2,128,1,float16,fp8,0,0.22318400939305624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,4,128,1,float16,float16,0,0.23638933897018433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,8,128,1,float16,fp8,0,0.2916586597760518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,4,128,1,float16,fp8,0,0.2767360011736552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,8,128,1,float16,float16,0,0.24223999182383218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,40,128,1,float16,float16,0,0.17634665966033936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,40,128,1,float16,fp8,0,0.2109066645304362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,1,128,1,float16,float16,0,0.1029866635799408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,1,128,1,float16,fp8,0,0.11693867047627766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,2,128,1,float16,float16,0,0.11185600360234578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,2,128,1,float16,fp8,0,0.12453333536783855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,4,128,1,float16,float16,0,0.12573867042859396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,8,128,1,float16,float16,0,0.13005866607030234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,4,128,1,float16,fp8,0,0.14983466267585754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,8,128,1,float16,fp8,0,0.1567146678765615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,40,128,1,float16,float16,0,0.09912533561388652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,40,128,1,float16,fp8,0,0.11987200379371643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,1,128,1,float16,float16,0,0.05893866717815399
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,1,128,1,float16,fp8,0,0.064751997590065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,2,128,1,float16,float16,0,0.06407466530799866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,2,128,1,float16,fp8,0,0.0721919983625412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,4,128,1,float16,float16,0,0.07272000114123027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,40,128,1,float16,float16,0,0.05700799822807312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,4,128,1,float16,fp8,0,0.08529067039489746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,8,128,1,float16,float16,0,0.07388799885908763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,8,128,1,float16,fp8,0,0.08861866593360901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,40,128,1,float16,fp8,0,0.07035733262697856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,1,128,1,float16,float16,0,0.03503466645876566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,2,128,1,float16,float16,0,0.03630933413902918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,1,128,1,float16,fp8,0,0.04138133426507314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,2,128,1,float16,fp8,0,0.04270400106906891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,4,128,1,float16,float16,0,0.039605334401130676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,4,128,1,float16,fp8,0,0.04640533526738485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,8,128,1,float16,float16,0,0.03996799886226654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,8,128,1,float16,fp8,0,0.04696000119050344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,40,128,1,float16,float16,0,0.031018666923046112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,40,128,1,float16,fp8,0,0.037077332536379494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,1,128,1,float16,float16,0,0.02535466601451238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,1,128,1,float16,fp8,0,0.02998399982849757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,2,128,1,float16,float16,0,0.025424001117547352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,2,128,1,float16,fp8,0,0.03046933313210805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,4,128,1,float16,float16,0,0.026549334327379864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,4,128,1,float16,fp8,0,0.03259200106064478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,8,128,1,float16,fp8,0,0.0329120010137558
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,8,128,1,float16,float16,0,0.026613332331180573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,1,128,1,float16,float16,0,1.1073226928710938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,2,128,1,float16,float16,0,1.246656020482381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,1,128,1,float16,fp8,0,1.2936000029246013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,2,128,1,float16,fp8,0,1.3715839385986328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,4,128,1,float16,float16,0,1.52402130762736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,4,128,1,float16,fp8,0,1.8085974057515461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,8,128,1,float16,float16,0,1.5302027066548665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,40,128,1,float16,float16,0,1.1391572952270508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,1,128,1,float16,float16,0,0.565392017364502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,40,128,1,float16,fp8,0,1.3061227003733318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,1,128,1,float16,fp8,0,0.65829865137736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,2,128,1,float16,float16,0,0.6280479828516642
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,2,128,1,float16,fp8,0,0.6968106428782145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,40,128,1,float16,float16,0,0.5771306753158569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,4,128,1,float16,fp8,0,0.9221813678741455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,8,128,1,float16,fp8,0,1.8061280250549316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,8,128,1,float16,float16,0,0.794549306233724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,1,128,1,float16,float16,0,0.29287999868392944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,40,128,1,float16,fp8,0,0.6725546518961588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,4,128,1,float16,float16,0,0.771125316619873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,8,128,1,float16,fp8,0,0.9429919719696045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,1,128,1,float16,fp8,0,0.3429280122121175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,2,128,1,float16,float16,0,0.33079999685287476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,4,128,1,float16,fp8,0,0.4697759946187337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,4,128,1,float16,float16,0,0.3964853286743164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,8,128,1,float16,fp8,0,0.4849439859390259
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,2,128,1,float16,fp8,0,0.36424533526102704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,8,128,1,float16,float16,0,0.40635732809702557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,40,128,1,float16,fp8,0,0.35013866424560547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,40,128,1,float16,float16,0,0.30108267068862915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,1,128,1,float16,float16,0,0.1567520002524058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,1,128,1,float16,fp8,0,0.18152532974878946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,2,128,1,float16,float16,0,0.17314666509628296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,2,128,1,float16,fp8,0,0.19272534052530924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,4,128,1,float16,float16,0,0.20777066548665366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,8,128,1,float16,fp8,0,0.256117324034373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,4,128,1,float16,fp8,0,0.24851200977961221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,40,128,1,float16,fp8,0,0.18708266814549765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,40,128,1,float16,float16,0,0.16006400187810263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,1,128,1,float16,float16,0,0.08753599723180135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,8,128,1,float16,float16,0,0.21522667010625204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,1,128,1,float16,fp8,0,0.10191999872525533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,4,128,1,float16,fp8,0,0.1342026690642039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,8,128,1,float16,float16,0,0.11543466647466023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,2,128,1,float16,float16,0,0.09559466441472371
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,2,128,1,float16,fp8,0,0.10902933279673259
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,4,128,1,float16,float16,0,0.11210133632024129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,40,128,1,float16,float16,0,0.0902880032857259
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,8,128,1,float16,fp8,0,0.1381706694761912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,40,128,1,float16,fp8,0,0.1055519978205363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,1,128,1,float16,float16,0,0.050010666251182556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,1,128,1,float16,fp8,0,0.05696000158786774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,2,128,1,float16,float16,0,0.05469333132108053
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,2,128,1,float16,fp8,0,0.06239999830722809
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,4,128,1,float16,float16,0,0.06413333117961884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,4,128,1,float16,fp8,0,0.075162669022878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,8,128,1,float16,float16,0,0.06515199939409892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,8,128,1,float16,fp8,0,0.07932266592979431
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,1,128,1,float16,fp8,0,0.03506666670242945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,40,128,1,float16,float16,0,0.05299733579158783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,40,128,1,float16,fp8,0,0.06504533191521962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,1,128,1,float16,float16,0,0.028954667349656422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,2,128,1,float16,float16,0,0.03200533241033554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,2,128,1,float16,fp8,0,0.03588266670703888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,4,128,1,float16,float16,0,0.03329599897066752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,4,128,1,float16,fp8,0,0.040847999354203544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,8,128,1,float16,float16,0,0.034559999903043113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,8,128,1,float16,fp8,0,0.04164800047874451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,40,128,1,float16,float16,0,0.027488000690937042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,40,128,1,float16,fp8,0,0.03456533451875051
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,1,128,1,float16,float16,0,0.02195200075705846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,1,128,1,float16,fp8,0,0.026549334327379864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,2,128,1,float16,float16,0,0.022367998957633972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,2,128,1,float16,fp8,0,0.026586666703224182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,4,128,1,float16,float16,0,0.02334933231274287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,8,128,1,float16,float16,0,0.023797333240509033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,4,128,1,float16,fp8,0,0.02888533224662145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,8,128,1,float16,fp8,0,0.029098667204380035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,40,128,1,float16,float16,0,0.018485333770513535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,40,128,1,float16,fp8,0,0.024175999065240223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,1,128,1,float16,float16,0,0.016794666647911072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,1,128,1,float16,fp8,0,0.01982933282852173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,2,128,1,float16,float16,0,0.017071999609470367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,2,128,1,float16,fp8,0,0.019509332875410717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,4,128,1,float16,float16,0,0.017456000049908955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,4,128,1,float16,fp8,0,0.021055998901526134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,8,128,1,float16,float16,0,0.017301333447297413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,8,128,1,float16,fp8,0,0.021162666380405426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,1,128,1,float16,float16,0,0.5383733510971069
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,1,128,1,float16,fp8,0,0.5788480043411255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,2,128,1,float16,float16,0,0.6164746681849161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,4,128,1,float16,float16,0,0.74563201268514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,2,128,1,float16,fp8,0,0.6262720028559366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,40,128,1,float16,fp8,0,0.6084266503651937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,40,128,1,float16,float16,0,0.5546346505482992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,8,128,1,float16,fp8,0,0.8829013506571451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,1,128,1,float16,float16,0,0.2815306584040324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,1,128,1,float16,fp8,0,0.3041813373565674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,2,128,1,float16,float16,0,0.3142506678899129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,4,128,1,float16,fp8,0,0.8425813515981039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,2,128,1,float16,fp8,0,0.32451732953389484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,4,128,1,float16,float16,0,0.3832319974899292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,40,128,1,float16,float16,0,0.2879679997762044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,8,128,1,float16,float16,0,0.3909279902776082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,1,128,1,float16,float16,0,0.15059733390808105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,4,128,1,float16,fp8,0,0.43421868483225506
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,8,128,1,float16,float16,0,0.7647466659545898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,40,128,1,float16,fp8,0,0.3189866741498311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,8,128,1,float16,fp8,0,0.4561920166015625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,1,128,1,float16,fp8,0,0.16484799981117249
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,2,128,1,float16,float16,0,0.16756266355514526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,2,128,1,float16,fp8,0,0.18022932608922324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,4,128,1,float16,float16,0,0.2001333236694336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,4,128,1,float16,fp8,0,0.22950400908788046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,8,128,1,float16,float16,0,0.20383999745051065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,8,128,1,float16,fp8,0,0.23896000782648721
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,40,128,1,float16,float16,0,0.15571199854214987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,1,128,1,float16,float16,0,0.08260266482830048
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,40,128,1,float16,fp8,0,0.17272533973058066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,1,128,1,float16,fp8,0,0.09498666723569234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,2,128,1,float16,float16,0,0.09200533231099446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,2,128,1,float16,fp8,0,0.10280000170071919
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,4,128,1,float16,float16,0,0.10867733756701152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,40,128,1,float16,float16,0,0.0888426701227824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,4,128,1,float16,fp8,0,0.1251359979311625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,8,128,1,float16,float16,0,0.11229866743087769
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,40,128,1,float16,fp8,0,0.09531733393669128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,8,128,1,float16,fp8,0,0.12905599673589072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,1,128,1,float16,float16,0,0.04756799836953481
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,1,128,1,float16,fp8,0,0.05000533163547516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,2,128,1,float16,float16,0,0.052527998884518944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,4,128,1,float16,float16,0,0.0617386649052302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,2,128,1,float16,fp8,0,0.05754666527112325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,4,128,1,float16,fp8,0,0.06830933193365733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,8,128,1,float16,float16,0,0.06368533273537953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,8,128,1,float16,fp8,0,0.07167466481526692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,40,128,1,float16,float16,0,0.05099200208981832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,40,128,1,float16,fp8,0,0.05672533313433329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,1,128,1,float16,float16,0,0.02698666602373123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,1,128,1,float16,fp8,0,0.03126399964094162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,4,128,1,float16,fp8,0,0.03569599986076355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,2,128,1,float16,float16,0,0.02804800122976303
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,2,128,1,float16,fp8,0,0.032416000962257385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,4,128,1,float16,float16,0,0.031445334355036415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,8,128,1,float16,float16,0,0.033370666205883026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,1,128,1,float16,float16,0,0.02037866661945979
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,8,128,1,float16,fp8,0,0.03603733330965042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,40,128,1,float16,fp8,0,0.028901333610216778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,40,128,1,float16,float16,0,0.027056001126766205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,4,128,1,float16,float16,0,0.02197333425283432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,1,128,1,float16,fp8,0,0.02420266717672348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,2,128,1,float16,float16,0,0.021066665649414062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,2,128,1,float16,fp8,0,0.02390933285156886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,4,128,1,float16,fp8,0,0.025626666843891144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,40,128,1,float16,float16,0,0.017210666090250015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,8,128,1,float16,float16,0,0.02216000109910965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,8,128,1,float16,fp8,0,0.025802666942278545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,40,128,1,float16,fp8,0,0.019776000330845516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,1,128,1,float16,float16,0,0.01584533353646596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,1,128,1,float16,fp8,0,0.018090666582187016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,2,128,1,float16,float16,0,0.01545599972208341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,4,128,1,float16,float16,0,0.01588800052801768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,2,128,1,float16,fp8,0,0.018101333330074947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,4,128,1,float16,fp8,0,0.01814933369557063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,8,128,1,float16,float16,0,0.015813333292802174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,8,128,1,float16,fp8,0,0.018330667167901993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,40,128,1,float16,float16,0,0.016165333489576977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,40,128,1,float16,fp8,0,0.019146667172511418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,1,128,1,float16,float16,0,0.014933332800865173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,1,128,1,float16,fp8,0,0.017642666896184284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,2,128,1,float16,float16,0,0.015226667126019796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,2,128,1,float16,fp8,0,0.017786666750907898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,4,128,1,float16,float16,0,0.015749332805474598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,4,128,1,float16,fp8,0,0.017514667163292568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,8,128,1,float16,float16,0,0.015520000209410986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,8,128,1,float16,fp8,0,0.017535999417304993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,1,128,1,float16,fp8,0,0.30590399106343585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,2,128,1,float16,float16,0,0.3188426693280538
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,2,128,1,float16,fp8,0,0.3280266722043355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,4,128,1,float16,float16,0,0.3814133405685425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,1,128,1,float16,float16,0,0.2775839964548747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,4,128,1,float16,fp8,0,0.4281066656112671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,8,128,1,float16,float16,0,0.4209226767222087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,8,128,1,float16,fp8,0,0.47493334611256915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,40,128,1,float16,float16,0,0.31995199124018353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,1,128,1,float16,float16,0,0.14895466963450113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,1,128,1,float16,fp8,0,0.16738667090733847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,2,128,1,float16,float16,0,0.16620799899101257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,2,128,1,float16,fp8,0,0.17819732427597046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,4,128,1,float16,float16,0,0.20120000839233398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,8,128,1,float16,fp8,0,0.25144533316294354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,40,128,1,float16,float16,0,0.16782933473587036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,40,128,1,float16,fp8,0,0.16878400246302286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,1,128,1,float16,float16,0,0.0848533312479655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,4,128,1,float16,fp8,0,0.22875199715296426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,1,128,1,float16,fp8,0,0.09494933485984802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,8,128,1,float16,float16,0,0.2218773365020752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,2,128,1,float16,float16,0,0.09250133236249287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,4,128,1,float16,fp8,0,0.12497599919637044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,2,128,1,float16,fp8,0,0.10314133763313293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,8,128,1,float16,fp8,0,0.13859200477600098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,8,128,1,float16,float16,0,0.11875733733177185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,40,128,1,float16,fp8,0,0.09236799677213033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,4,128,1,float16,float16,0,0.10894933342933655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,1,128,1,float16,float16,0,0.04671466847260793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,2,128,1,float16,fp8,0,0.05592533449331919
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,40,128,1,float16,float16,0,0.09378666679064433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,1,128,1,float16,fp8,0,0.048197334011395775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,40,128,1,float16,fp8,0,0.31722132364908856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,2,128,1,float16,float16,0,0.05171733101209005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,4,128,1,float16,float16,0,0.06085866689682007
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,4,128,1,float16,fp8,0,0.06836266815662384
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,8,128,1,float16,float16,0,0.06659733255704244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,8,128,1,float16,fp8,0,0.07694933315118153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,40,128,1,float16,float16,0,0.05346666773160299
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,40,128,1,float16,fp8,0,0.04990933338801066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,1,128,1,float16,fp8,0,0.031082667410373688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,4,128,1,float16,fp8,0,0.03581333408753077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,2,128,1,float16,float16,0,0.02865600089232127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,1,128,1,float16,float16,0,0.026533332963784535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,2,128,1,float16,fp8,0,0.03242133309443792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,1,128,1,float16,float16,0,0.02077866718173027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,4,128,1,float16,float16,0,0.030405332644780476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,8,128,1,float16,float16,0,0.031685332457224526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,8,128,1,float16,fp8,0,0.035973332822322845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,40,128,1,float16,float16,0,0.026101333399613697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,40,128,1,float16,fp8,0,0.0258240004380544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,4,128,1,float16,fp8,0,0.02571200082699458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,1,128,1,float16,fp8,0,0.0240639994541804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,2,128,1,float16,float16,0,0.0206986665725708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,2,128,1,float16,fp8,0,0.023925334215164185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,4,128,1,float16,float16,0,0.021946666141351063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,8,128,1,float16,float16,0,0.0220266655087471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,8,128,1,float16,fp8,0,0.025498665869235992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,40,128,1,float16,float16,0,0.018133333573738735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,40,128,1,float16,fp8,0,0.018863999595244724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,1,128,1,float16,float16,0,0.015306666493415833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,1,128,1,float16,fp8,0,0.018186666071414948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,2,128,1,float16,float16,0,0.015642666568358738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,2,128,1,float16,fp8,0,0.017711999515692394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,4,128,1,float16,float16,0,0.015930666277805965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,4,128,1,float16,fp8,0,0.018112000077962875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,8,128,1,float16,float16,0,0.015696000307798386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,8,128,1,float16,fp8,0,0.0180479995906353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,40,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,40,128,1,float16,fp8,0,0.014991999914248785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,1,128,1,float16,float16,0,0.01498666654030482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,1,128,1,float16,fp8,0,0.01777600000301997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,2,128,1,float16,float16,0,0.015381333728631338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,2,128,1,float16,fp8,0,0.018144000321626663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,4,128,1,float16,float16,0,0.01504533365368843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,8,128,1,float16,float16,0,0.015647999942302704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,4,128,1,float16,fp8,0,0.018053332964579265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,8,128,1,float16,fp8,0,0.017903999735911686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,40,128,1,float16,float16,0,0.013631999492645264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,40,128,1,float16,fp8,0,0.014688000082969666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,1,128,1,float16,float16,0,0.014639999717473984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,1,128,1,float16,fp8,0,0.017263999829689663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,2,128,1,float16,float16,0,0.015098666151364645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,2,128,1,float16,fp8,0,0.016869333883126576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,4,128,1,float16,float16,0,0.01470400020480156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,4,128,1,float16,fp8,0,0.01672533278663953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,8,128,1,float16,float16,0,0.014554666976133982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,8,128,1,float16,fp8,0,0.017407999684413273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,1,128,1,float16,float16,0,0.14897066354751587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,1,128,1,float16,fp8,0,0.16590399543444315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,2,128,1,float16,fp8,0,0.17795199155807495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,4,128,1,float16,float16,0,0.21502933899561563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,4,128,1,float16,fp8,0,0.24527466297149658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,8,128,1,float16,float16,0,0.2507359981536865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,8,128,1,float16,fp8,0,0.2903199990590413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,40,128,1,float16,fp8,0,0.2248213291168213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,1,128,1,float16,float16,0,0.08265066643555959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,2,128,1,float16,float16,0,0.1646719972292582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,1,128,1,float16,fp8,0,0.09602133433024089
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,2,128,1,float16,float16,0,0.0925546685854594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,2,128,1,float16,fp8,0,0.10207999746004741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,4,128,1,float16,float16,0,0.11588799953460693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,8,128,1,float16,float16,0,0.13269333044687906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,4,128,1,float16,fp8,0,0.13506133357683817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,40,128,1,float16,float16,0,0.12197333574295044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,40,128,1,float16,fp8,0,0.11988266309102376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,8,128,1,float16,fp8,0,0.15897599856058756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,2,128,1,float16,float16,0,0.051728000243504844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,1,128,1,float16,float16,0,0.04771199822425842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,1,128,1,float16,fp8,0,0.05130666494369507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,2,128,1,float16,fp8,0,0.055349335074424744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,4,128,1,float16,float16,0,0.06592000027497609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,8,128,1,float16,fp8,0,0.08763200044631958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,40,128,1,float16,float16,0,0.2328480084737142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,4,128,1,float16,fp8,0,0.07363200187683105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,8,128,1,float16,float16,0,0.0754559983809789
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,40,128,1,float16,float16,0,0.06815466781457265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,40,128,1,float16,fp8,0,0.06517333288987477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,1,128,1,float16,float16,0,0.026943999032179516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,1,128,1,float16,fp8,0,0.03068266560633977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,2,128,1,float16,float16,0,0.02758399893840154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,8,128,1,float16,float16,0,0.03645866612593333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,4,128,1,float16,float16,0,0.030479999879995983
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,2,128,1,float16,fp8,0,0.032218667368094124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,4,128,1,float16,fp8,0,0.03580799947182337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,8,128,1,float16,fp8,0,0.04154133299986521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,40,128,1,float16,float16,0,0.03339199970165888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,40,128,1,float16,fp8,0,0.031727999448776245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,1,128,1,float16,float16,0,0.020432000358899433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,4,128,1,float16,fp8,0,0.025402667621771496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,1,128,1,float16,fp8,0,0.02386666586001714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,2,128,1,float16,float16,0,0.020576000213623047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,2,128,1,float16,fp8,0,0.023845332364241283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,4,128,1,float16,float16,0,0.021882665654023487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,8,128,1,float16,float16,0,0.022229333718617756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,8,128,1,float16,fp8,0,0.025631998976071674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,40,128,1,float16,float16,0,0.021290667355060577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,40,128,1,float16,fp8,0,0.02160533269246419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,1,128,1,float16,float16,0,0.01544533297419548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,1,128,1,float16,fp8,0,0.01783466711640358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,2,128,1,float16,float16,0,0.015541333705186844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,2,128,1,float16,fp8,0,0.01807466646035512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,40,128,1,float16,float16,0,0.01695466662446658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,4,128,1,float16,float16,0,0.016048000504573185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,4,128,1,float16,fp8,0,0.018031999468803406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,8,128,1,float16,float16,0,0.01544533297419548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,8,128,1,float16,fp8,0,0.017680000513792038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,40,128,1,float16,fp8,0,0.01777600000301997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,1,128,1,float16,float16,0,0.01553600033124288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,1,128,1,float16,fp8,0,0.0176959993938605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,2,128,1,float16,float16,0,0.015295999745527903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,2,128,1,float16,fp8,0,0.017898666361967724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,4,128,1,float16,float16,0,0.015365333606799444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,4,128,1,float16,fp8,0,0.017808000246683758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,8,128,1,float16,float16,0,0.014949332922697067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,8,128,1,float16,fp8,0,0.017530667285124462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,40,128,1,float16,float16,0,0.013973332941532135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,40,128,1,float16,fp8,0,0.014805333067973455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,1,128,1,float16,float16,0,0.01469333345691363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,1,128,1,float16,fp8,0,0.01684800038735072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,2,128,1,float16,float16,0,0.01458666721979777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,2,128,1,float16,fp8,0,0.016842667013406754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,4,128,1,float16,float16,0,0.014885333677132925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,4,128,1,float16,fp8,0,0.01722666621208191
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,8,128,1,float16,float16,0,0.015306666493415833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,8,128,1,float16,fp8,0,0.01708799973130226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,40,128,1,float16,float16,0,0.013637332866589228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,40,128,1,float16,fp8,0,0.014592000593741735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,1,128,1,float16,float16,0,0.014208000153303146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,1,128,1,float16,fp8,0,0.016069332758585613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,2,128,1,float16,float16,0,0.014064000298579534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,2,128,1,float16,fp8,0,0.016458666572968166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,4,128,1,float16,float16,0,0.014186666657527288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,4,128,1,float16,fp8,0,0.016234666109085083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,8,128,1,float16,float16,0,0.01431999976436297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,8,128,1,float16,fp8,0,0.01661866654952367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,1,128,1,float16,float16,0,0.08373333017031352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,1,128,1,float16,fp8,0,0.09564266602198283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,2,128,1,float16,float16,0,0.0992746651172638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,2,128,1,float16,fp8,0,0.1120799978574117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,4,128,1,float16,float16,0,0.1302826702594757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,4,128,1,float16,fp8,0,0.15500799814860025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,8,128,1,float16,float16,0,0.13545067111651102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,40,128,1,float16,float16,0,0.18708799282709757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,8,128,1,float16,fp8,0,0.159360001484553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,1,128,1,float16,float16,0,0.04762666424115499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,40,128,1,float16,fp8,0,0.1830079952875773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,1,128,1,float16,fp8,0,0.05081599950790405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,2,128,1,float16,float16,0,0.05710933109124502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,2,128,1,float16,fp8,0,0.06044800082842509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,4,128,1,float16,float16,0,0.07355733215808868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,4,128,1,float16,fp8,0,0.08301333089669545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,8,128,1,float16,float16,0,0.07785066465536754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,40,128,1,float16,float16,0,0.09938666224479675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,40,128,1,float16,fp8,0,0.09624532858530681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,8,128,1,float16,fp8,0,0.08754133184750874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,1,128,1,float16,float16,0,0.02643733223279317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,1,128,1,float16,fp8,0,0.0308746670683225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,2,128,1,float16,float16,0,0.02735999971628189
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,2,128,1,float16,fp8,0,0.03215999901294708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,4,128,1,float16,float16,0,0.035504000882307686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,4,128,1,float16,fp8,0,0.04106666644414266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,8,128,1,float16,float16,0,0.035391998787721
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,8,128,1,float16,fp8,0,0.04101866732041041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,40,128,1,float16,float16,0,0.049866666396458946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,40,128,1,float16,fp8,0,0.04623466730117798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,2,128,1,float16,fp8,0,0.023711999257405598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,1,128,1,float16,float16,0,0.020303999384244282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,8,128,1,float16,float16,0,0.021712000171343487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,1,128,1,float16,fp8,0,0.023872000475724537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,8,128,1,float16,fp8,0,0.025402667621771496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,2,128,1,float16,float16,0,0.020373333245515823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,4,128,1,float16,fp8,0,0.025813333690166473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,4,128,1,float16,float16,0,0.022106667359670002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,40,128,1,float16,float16,0,0.028079998989899952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,40,128,1,float16,fp8,0,0.028746667007605236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,1,128,1,float16,float16,0,0.015344000111023584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,1,128,1,float16,fp8,0,0.017674667139848072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,2,128,1,float16,float16,0,0.01533866673707962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,2,128,1,float16,fp8,0,0.017797333498795826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,4,128,1,float16,float16,0,0.015466666469971338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,4,128,1,float16,fp8,0,0.018005333840847015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,8,128,1,float16,float16,0,0.015295999745527903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,8,128,1,float16,fp8,0,0.017994667092959087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,40,128,1,float16,float16,0,0.02054399996995926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,40,128,1,float16,fp8,0,0.021359999974568684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,1,128,1,float16,float16,0,0.015279999623696009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,1,128,1,float16,fp8,0,0.017477333545684814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,2,128,1,float16,float16,0,0.01525866612792015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,2,128,1,float16,fp8,0,0.01785600061217944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,4,128,1,float16,float16,0,0.015087999403476715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,4,128,1,float16,fp8,0,0.017994667092959087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,8,128,1,float16,float16,0,0.014864000181357065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,8,128,1,float16,fp8,0,0.017727999637524288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,2,128,1,float16,float16,0,0.014943999548753103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,2,128,1,float16,fp8,0,0.017344000438849132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,40,128,1,float16,float16,0,0.01647466669480006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,40,128,1,float16,fp8,0,0.01773333301146825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,1,128,1,float16,float16,0,0.014650666465361914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,1,128,1,float16,fp8,0,0.016778666526079178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,4,128,1,float16,float16,0,0.014874666929244995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,4,128,1,float16,fp8,0,0.01757866640885671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,8,128,1,float16,float16,0,0.014933332800865173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,8,128,1,float16,fp8,0,0.017114666601022083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,40,128,1,float16,float16,0,0.013749333719412485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,40,128,1,float16,fp8,0,0.01471466695268949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,1,128,1,float16,float16,0,0.014197333405415217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,8,128,1,float16,float16,0,0.013983999689420065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,1,128,1,float16,fp8,0,0.0161920003592968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,2,128,1,float16,float16,0,0.014042666802803675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,2,128,1,float16,fp8,0,0.016224000602960587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,4,128,1,float16,float16,0,0.01444799949725469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,4,128,1,float16,fp8,0,0.016575999557971954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,8,128,1,float16,fp8,0,0.01600533351302147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,40,128,1,float16,float16,0,0.013690666606028875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,40,128,1,float16,fp8,0,0.014538666854302088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,1,128,1,float16,float16,0,0.013909333695967993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,1,128,1,float16,fp8,0,0.016186666985352833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,2,128,1,float16,float16,0,0.013925333817799887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,2,128,1,float16,fp8,0,0.01613333324591319
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,4,128,1,float16,float16,0,0.014074667046467463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,4,128,1,float16,fp8,0,0.015957333147525787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,8,128,1,float16,float16,0,0.013951999445756277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,8,128,1,float16,fp8,0,0.015834666788578033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,1,128,1,float16,fp8,0,27.300875345865887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,2,128,1,float16,fp8,0,27.453898111979168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,1,128,1,float16,float16,0,30.15410614013672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,4,128,1,float16,fp8,0,28.382774353027344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,2,128,1,float16,float16,0,31.750869750976562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,4,128,1,float16,float16,0,30.48375956217448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,8,128,1,float16,float16,0,32.345375061035156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,32,128,1,float16,fp8,0,14.586634318033854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,1,128,1,float16,fp8,0,13.706293741861979
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,32,128,1,float16,float16,0,16.12659200032552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,1,128,1,float16,float16,0,15.304490407307943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,2,128,1,float16,fp8,0,13.949546813964844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,2,128,1,float16,float16,0,15.987637837727865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,8,128,1,float16,fp8,0,28.184127807617188
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,4,128,1,float16,float16,0,15.92514165242513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,32,128,1,float16,float16,0,8.29313596089681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,32,128,1,float16,fp8,0,7.432207743326823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,1,128,1,float16,float16,0,7.579429626464844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,1,128,1,float16,fp8,0,6.86027717590332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,2,128,1,float16,float16,0,7.698245366414388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,4,128,1,float16,fp8,0,14.245936075846354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,8,128,1,float16,fp8,0,14.145328521728516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,8,128,1,float16,float16,0,16.60095469156901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,2,128,1,float16,fp8,0,6.98039436340332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,4,128,1,float16,float16,0,7.968997319539388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,32,128,1,float16,float16,0,3.8130079905192056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,8,128,1,float16,float16,0,7.7201493581136065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,4,128,1,float16,fp8,0,7.5588531494140625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,32,128,1,float16,fp8,0,3.7117706934611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,1,128,1,float16,float16,0,3.617664019266764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,8,128,1,float16,fp8,0,7.185290654500325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,1,128,1,float16,fp8,0,3.520602544148763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,2,128,1,float16,float16,0,3.8917598724365234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,2,128,1,float16,fp8,0,3.6302239100138345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,4,128,1,float16,float16,0,3.7250986099243164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,4,128,1,float16,fp8,0,3.663018544514974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,8,128,1,float16,float16,0,3.688229242960612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,8,128,1,float16,fp8,0,3.944064140319824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,1,128,1,float16,fp8,0,15.657066345214844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,1,128,1,float16,float16,0,20.264458974202473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,2,128,1,float16,fp8,0,15.789199829101562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,4,128,1,float16,float16,0,17.39789326985677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,4,128,1,float16,fp8,0,16.450720469156902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,2,128,1,float16,float16,0,19.23788324991862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,8,128,1,float16,fp8,0,16.57687505086263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,8,128,1,float16,float16,0,18.313604990641277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,32,128,1,float16,fp8,0,8.547221501668295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,32,128,1,float16,float16,0,9.945007960001627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,1,128,1,float16,float16,0,8.918954849243164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,1,128,1,float16,fp8,0,7.8821760813395185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,2,128,1,float16,float16,0,9.201775868733725
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,2,128,1,float16,fp8,0,8.036613464355469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,4,128,1,float16,float16,0,8.919504165649414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,4,128,1,float16,fp8,0,8.266101201375326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,32,128,1,float16,float16,0,4.595610618591309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,1,128,1,float16,float16,0,4.211013476053874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,32,128,1,float16,fp8,0,4.320506731669108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,1,128,1,float16,fp8,0,3.987002690633138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,2,128,1,float16,float16,0,4.269791920979817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,2,128,1,float16,fp8,0,3.987722714742025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,8,128,1,float16,float16,0,9.291130701700846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,8,128,1,float16,fp8,0,8.26909319559733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,32,128,1,float16,float16,0,2.2320693333943686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,32,128,1,float16,fp8,0,2.220282713572184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,4,128,1,float16,float16,0,4.129184087117513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,8,128,1,float16,float16,0,4.250250816345215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,1,128,1,float16,float16,0,2.1841279665629068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,1,128,1,float16,fp8,0,2.036693255106608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,4,128,1,float16,fp8,0,4.45578670501709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,8,128,1,float16,fp8,0,4.174538612365723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,2,128,1,float16,float16,0,2.0854719479878745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,2,128,1,float16,fp8,0,2.033733367919922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,4,128,1,float16,float16,0,2.084890683492025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,8,128,1,float16,float16,0,2.153114636739095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,4,128,1,float16,fp8,0,2.3431413968404136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,8,128,1,float16,fp8,0,2.1285120646158853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,1,128,1,float16,fp8,0,11.232789357503256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,1,128,1,float16,float16,0,12.32302983601888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,2,128,1,float16,fp8,0,11.144842783610025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,2,128,1,float16,float16,0,12.744283040364584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,4,128,1,float16,float16,0,12.624608357747396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,4,128,1,float16,fp8,0,11.775775909423828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,8,128,1,float16,fp8,0,11.597829182942709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,8,128,1,float16,float16,0,12.919610341389975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,32,128,1,float16,float16,0,6.4440053304036455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,1,128,1,float16,float16,0,5.961589177449544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,32,128,1,float16,fp8,0,6.324517567952474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,1,128,1,float16,fp8,0,5.583802541097005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,2,128,1,float16,float16,0,5.987749099731445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,2,128,1,float16,fp8,0,5.608453114827474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,4,128,1,float16,float16,0,6.30839474995931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,4,128,1,float16,fp8,0,5.9960479736328125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,32,128,1,float16,float16,0,3.1904640197753906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,32,128,1,float16,fp8,0,3.1110452016194663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,1,128,1,float16,float16,0,3.008549372355143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,1,128,1,float16,fp8,0,2.814901351928711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,2,128,1,float16,float16,0,2.8950026830037436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,8,128,1,float16,float16,0,6.557653427124023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,8,128,1,float16,fp8,0,6.08348274230957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,2,128,1,float16,fp8,0,2.8498827616373696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,4,128,1,float16,fp8,0,2.9647038777669272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,4,128,1,float16,float16,0,2.9282665252685547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,8,128,1,float16,float16,0,3.096954663594564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,32,128,1,float16,float16,0,1.6424533526102703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,32,128,1,float16,fp8,0,1.6053493817647297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,1,128,1,float16,float16,0,1.4404053688049316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,1,128,1,float16,fp8,0,1.4473919868469238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,8,128,1,float16,fp8,0,3.172783851623535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,2,128,1,float16,float16,0,1.4959306716918945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,2,128,1,float16,fp8,0,1.521407922108968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,4,128,1,float16,float16,0,1.5109546979268391
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,4,128,1,float16,fp8,0,1.5347839991251628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,8,128,1,float16,float16,0,1.5176587104797363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,8,128,1,float16,fp8,0,1.5243412653605144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,1,128,1,float16,fp8,0,14.46017583211263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,1,128,1,float16,float16,0,15.76962661743164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,2,128,1,float16,fp8,0,14.696586608886719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,2,128,1,float16,float16,0,16.587759653727215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,4,128,1,float16,float16,0,16.46075693766276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,4,128,1,float16,fp8,0,15.409781138102213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,8,128,1,float16,float16,0,16.782506306966145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,32,128,1,float16,float16,0,8.89677874247233
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,1,128,1,float16,float16,0,8.33901341756185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,32,128,1,float16,fp8,0,8.155653635660807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,1,128,1,float16,fp8,0,7.239744186401367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,2,128,1,float16,float16,0,7.9051361083984375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,2,128,1,float16,fp8,0,7.292805353800456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,8,128,1,float16,fp8,0,15.31067148844401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,4,128,1,float16,float16,0,8.24939219156901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,32,128,1,float16,float16,0,4.218015988667806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,1,128,1,float16,float16,0,3.6241280237833657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,32,128,1,float16,fp8,0,4.349098523457845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,1,128,1,float16,fp8,0,3.638474782307943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,2,128,1,float16,float16,0,3.8947626749674478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,4,128,1,float16,fp8,0,7.839989344278972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,8,128,1,float16,fp8,0,7.668287913004558
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,8,128,1,float16,float16,0,8.488293329874674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,2,128,1,float16,fp8,0,3.919658660888672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,4,128,1,float16,float16,0,3.8485546112060547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,4,128,1,float16,fp8,0,3.9012533823649087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,32,128,1,float16,float16,0,2.09554131825765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,8,128,1,float16,float16,0,3.9284693400065103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,1,128,1,float16,float16,0,1.948245366414388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,8,128,1,float16,fp8,0,4.0033919016520185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,32,128,1,float16,fp8,0,2.0966080029805503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,1,128,1,float16,fp8,0,1.9065225919087727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,2,128,1,float16,float16,0,1.9195094108581543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,2,128,1,float16,fp8,0,1.886298656463623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,4,128,1,float16,float16,0,1.9302453994750977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,4,128,1,float16,fp8,0,2.0978399912516275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,8,128,1,float16,float16,0,2.029088020324707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,1,128,1,float16,float16,0,0.9780960083007812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,32,128,1,float16,float16,0,1.075818697611491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,32,128,1,float16,fp8,0,1.111967960993449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,2,128,1,float16,float16,0,0.9837386608123779
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,8,128,1,float16,fp8,0,1.989034652709961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,1,128,1,float16,fp8,0,0.9632426897684733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,2,128,1,float16,fp8,0,1.0172053178151448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,4,128,1,float16,float16,0,1.0279040336608887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,4,128,1,float16,fp8,0,1.104202667872111
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,8,128,1,float16,float16,0,1.0130720138549805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,8,128,1,float16,fp8,0,1.0577387015024822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,1,128,1,float16,fp8,0,8.403674443562826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,1,128,1,float16,float16,0,9.232170740763346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,2,128,1,float16,float16,0,9.385786692301432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,2,128,1,float16,fp8,0,8.561349232991537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,4,128,1,float16,float16,0,9.861637115478516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,4,128,1,float16,fp8,0,9.253077189127604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,8,128,1,float16,float16,0,9.799999872843424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,8,128,1,float16,fp8,0,9.148197174072266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,32,128,1,float16,float16,0,4.876693407694499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,1,128,1,float16,float16,0,4.364426612854004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,1,128,1,float16,fp8,0,4.226949373881022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,32,128,1,float16,fp8,0,4.964277267456055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,2,128,1,float16,float16,0,4.543306668599446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,2,128,1,float16,fp8,0,4.290037473042806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,4,128,1,float16,fp8,0,4.572592099507649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,4,128,1,float16,float16,0,4.677221298217773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,32,128,1,float16,float16,0,2.472592035929362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,32,128,1,float16,fp8,0,2.481488068898519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,8,128,1,float16,float16,0,4.679093360900879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,1,128,1,float16,float16,0,2.1818453470865884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,8,128,1,float16,fp8,0,4.60093879699707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,1,128,1,float16,fp8,0,2.1470239957173667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,2,128,1,float16,float16,0,2.187338670094808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,2,128,1,float16,fp8,0,2.212277412414551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,4,128,1,float16,float16,0,2.24345064163208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,32,128,1,float16,float16,0,1.2468480269114177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,4,128,1,float16,fp8,0,2.32370662689209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,8,128,1,float16,float16,0,2.333301385243734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,32,128,1,float16,fp8,0,1.2756799856821697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,1,128,1,float16,float16,0,1.0927093029022217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,2,128,1,float16,float16,0,1.1188960075378418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,8,128,1,float16,fp8,0,2.4390506744384766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,1,128,1,float16,fp8,0,1.104314645131429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,2,128,1,float16,fp8,0,1.1205546855926514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,4,128,1,float16,float16,0,1.1409760316212971
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,4,128,1,float16,fp8,0,1.1973493099212646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,8,128,1,float16,float16,0,1.1615946292877197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,32,128,1,float16,float16,0,0.6595840056737264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,8,128,1,float16,fp8,0,1.2293333212534587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,32,128,1,float16,fp8,0,0.6745493412017822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,2,128,1,float16,fp8,0,0.5908213456471761
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,1,128,1,float16,float16,0,0.5627146561940511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,1,128,1,float16,fp8,0,0.5959306557973226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,4,128,1,float16,fp8,0,0.6399413347244263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,8,128,1,float16,float16,0,0.6261013348897299
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,2,128,1,float16,float16,0,0.5702559947967529
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,4,128,1,float16,float16,0,0.5936479965845743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,8,128,1,float16,fp8,0,0.6346880197525024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,1,128,1,float16,float16,0,8.661322911580404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,1,128,1,float16,fp8,0,8.119183858235678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,2,128,1,float16,float16,0,8.468037287394205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,2,128,1,float16,fp8,0,8.292832056681315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,4,128,1,float16,float16,0,8.858778635660807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,4,128,1,float16,fp8,0,8.839626948038736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,8,128,1,float16,float16,0,9.189178466796875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,32,128,1,float16,float16,0,4.825791994730632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,1,128,1,float16,float16,0,4.146373430887858
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,32,128,1,float16,fp8,0,5.216965357462565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,1,128,1,float16,fp8,0,4.726192156473796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,2,128,1,float16,float16,0,4.327706654866536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,2,128,1,float16,fp8,0,4.105786641438802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,8,128,1,float16,fp8,0,8.897130966186523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,4,128,1,float16,float16,0,4.33732795715332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,32,128,1,float16,float16,0,2.4033652941385903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,1,128,1,float16,float16,0,1.9666293462117512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,32,128,1,float16,fp8,0,2.5955467224121094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,4,128,1,float16,fp8,0,4.648288091023763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,8,128,1,float16,float16,0,4.545546531677246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,8,128,1,float16,fp8,0,4.464298566182454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,1,128,1,float16,fp8,0,2.0191680590311685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,2,128,1,float16,fp8,0,2.074186642964681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,4,128,1,float16,float16,0,2.134981314341227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,2,128,1,float16,float16,0,2.2547359466552734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,4,128,1,float16,fp8,0,2.315440018971761
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,32,128,1,float16,float16,0,1.2316160202026367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,8,128,1,float16,float16,0,2.246767997741699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,32,128,1,float16,fp8,0,1.2535999615987141
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,8,128,1,float16,fp8,0,2.392848014831543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,1,128,1,float16,float16,0,1.0066133340199788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,2,128,1,float16,float16,0,1.0497386455535889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,1,128,1,float16,fp8,0,1.0877546469370525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,2,128,1,float16,fp8,0,1.0813653469085693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,4,128,1,float16,float16,0,1.0881120363871257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,4,128,1,float16,fp8,0,1.1745920181274414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,8,128,1,float16,float16,0,1.1251200040181477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,8,128,1,float16,fp8,0,1.186405340830485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,32,128,1,float16,float16,0,0.6357119878133138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,32,128,1,float16,fp8,0,0.6901333332061768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,1,128,1,float16,float16,0,0.5206986665725708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,2,128,1,float16,float16,0,0.5417600075403849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,1,128,1,float16,fp8,0,0.552512009938558
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,2,128,1,float16,fp8,0,0.5673066775004069
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,4,128,1,float16,float16,0,0.5551146666208903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,4,128,1,float16,fp8,0,0.6089280049006144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,32,128,1,float16,fp8,0,0.3646933237711589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,8,128,1,float16,fp8,0,0.624069333076477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,8,128,1,float16,float16,0,0.584336002667745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,32,128,1,float16,float16,0,0.3397386471430461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,1,128,1,float16,float16,0,0.276853342851003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,1,128,1,float16,fp8,0,0.2993333339691162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,2,128,1,float16,float16,0,0.28489067157109577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,2,128,1,float16,fp8,0,0.3066026568412781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,4,128,1,float16,float16,0,0.3009546597798665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,4,128,1,float16,fp8,0,0.3272479971249898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,8,128,1,float16,float16,0,0.3138879934946696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,8,128,1,float16,fp8,0,0.3382879892985026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,1,128,1,float16,fp8,0,4.799930572509766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,2,128,1,float16,float16,0,4.897861480712891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,2,128,1,float16,fp8,0,4.945407867431641
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,1,128,1,float16,float16,0,4.861173311869304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,4,128,1,float16,float16,0,5.152960141499837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,8,128,1,float16,float16,0,5.410570780436198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,8,128,1,float16,fp8,0,5.563194910685222
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,4,128,1,float16,fp8,0,5.4699147542317705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,1,128,1,float16,float16,0,2.3829280535380044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,32,128,1,float16,float16,0,2.9913867314656577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,32,128,1,float16,fp8,0,3.3630825678507485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,2,128,1,float16,float16,0,2.467205365498861
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,1,128,1,float16,fp8,0,2.5639893213907876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,2,128,1,float16,fp8,0,2.6104960441589355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,4,128,1,float16,float16,0,2.5719359715779624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,4,128,1,float16,fp8,0,2.735328038533529
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,32,128,1,float16,float16,0,1.5134506225585938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,1,128,1,float16,float16,0,1.1878986358642578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,32,128,1,float16,fp8,0,1.6510079701741536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,8,128,1,float16,float16,0,2.7345546086629233
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,1,128,1,float16,fp8,0,1.22979736328125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,2,128,1,float16,float16,0,1.2918293476104736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,8,128,1,float16,fp8,0,2.8455626169840493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,2,128,1,float16,fp8,0,1.2577653725941975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,4,128,1,float16,float16,0,1.3165226777394612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,4,128,1,float16,fp8,0,1.4417492548624675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,32,128,1,float16,float16,0,0.7613440354665121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,8,128,1,float16,float16,0,1.3711360295613606
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,8,128,1,float16,fp8,0,1.4498133659362793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,32,128,1,float16,fp8,0,0.8322292963663737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,1,128,1,float16,fp8,0,0.6332213481267294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,1,128,1,float16,float16,0,0.6085386673609415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,2,128,1,float16,float16,0,0.63646399974823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,2,128,1,float16,fp8,0,0.662559986114502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,4,128,1,float16,float16,0,0.6615306536356608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,4,128,1,float16,fp8,0,0.7300586700439453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,32,128,1,float16,float16,0,0.403872013092041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,32,128,1,float16,fp8,0,0.43494399388631183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,1,128,1,float16,float16,0,0.31937599182128906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,8,128,1,float16,float16,0,0.696832021077474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,8,128,1,float16,fp8,0,0.7280213038126627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,1,128,1,float16,fp8,0,0.3389706611633301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,2,128,1,float16,float16,0,0.3378880023956299
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,2,128,1,float16,fp8,0,0.35286398728688556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,4,128,1,float16,float16,0,0.3521173397699992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,4,128,1,float16,fp8,0,0.38913599650065106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,8,128,1,float16,float16,0,0.36661867300669354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,8,128,1,float16,fp8,0,0.39023999373118085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,32,128,1,float16,float16,0,0.21906665960947672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,32,128,1,float16,fp8,0,0.24009066820144653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,1,128,1,float16,float16,0,0.17683732509613037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,1,128,1,float16,fp8,0,0.18958399693171182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,2,128,1,float16,float16,0,0.1850773294766744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,2,128,1,float16,fp8,0,0.19829867283503214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,4,128,1,float16,float16,0,0.18678400913874307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,8,128,1,float16,float16,0,0.2052853306134542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,4,128,1,float16,fp8,0,0.20506133635838827
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,8,128,1,float16,fp8,0,0.22170666853586832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,1,128,1,float16,float16,0,4.546101252237956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,1,128,1,float16,fp8,0,4.818613370259603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,4,128,1,float16,float16,0,5.431519826253255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,2,128,1,float16,fp8,0,4.921226819356282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,8,128,1,float16,float16,0,5.535946528116862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,4,128,1,float16,fp8,0,5.714303970336914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,2,128,1,float16,float16,0,4.913333257039388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,32,128,1,float16,fp8,0,3.357450803120931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,1,128,1,float16,fp8,0,2.47816530863444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,1,128,1,float16,float16,0,2.2594666481018066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,32,128,1,float16,float16,0,3.126688003540039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,2,128,1,float16,float16,0,2.362816015879313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,8,128,1,float16,fp8,0,5.814064025878906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,2,128,1,float16,fp8,0,2.510965347290039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,4,128,1,float16,float16,0,2.607797304789225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,4,128,1,float16,fp8,0,2.8635520935058594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,32,128,1,float16,float16,0,1.580143928527832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,8,128,1,float16,float16,0,2.7569707234700522
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,32,128,1,float16,fp8,0,1.7594079971313477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,1,128,1,float16,float16,0,1.1423947016398113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,8,128,1,float16,fp8,0,2.90777587890625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,2,128,1,float16,fp8,0,1.2770986557006836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,4,128,1,float16,float16,0,1.3230613072713215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,1,128,1,float16,fp8,0,1.2204266389211018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,2,128,1,float16,float16,0,1.210479974746704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,4,128,1,float16,fp8,0,1.463941256205241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,8,128,1,float16,float16,0,1.3913812637329102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,8,128,1,float16,fp8,0,1.4757653872172039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,32,128,1,float16,float16,0,0.7992853323618571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,1,128,1,float16,float16,0,0.5882346630096436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,1,128,1,float16,fp8,0,0.6290880044301351
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,32,128,1,float16,fp8,0,0.8491413593292236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,2,128,1,float16,float16,0,0.6241973241170248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,2,128,1,float16,fp8,0,0.6611146529515585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,4,128,1,float16,float16,0,0.6742933591206869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,4,128,1,float16,fp8,0,0.7347626686096191
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,8,128,1,float16,float16,0,0.7056799729665121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,8,128,1,float16,fp8,0,0.7669493357340494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,32,128,1,float16,float16,0,0.41468266646067303
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,32,128,1,float16,fp8,0,0.44193601608276367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,1,128,1,float16,float16,0,0.3104106585184733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,1,128,1,float16,fp8,0,0.3335733413696289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,2,128,1,float16,float16,0,0.3280533353487651
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,2,128,1,float16,fp8,0,0.3424266576766968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,4,128,1,float16,float16,0,0.35177067915598553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,4,128,1,float16,fp8,0,0.3888853391011556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,8,128,1,float16,float16,0,0.3661493460337321
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,8,128,1,float16,fp8,0,0.3999306758244832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,32,128,1,float16,float16,0,0.2248693307240804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,32,128,1,float16,fp8,0,0.24106132984161377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,1,128,1,float16,float16,0,0.16794667641321817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,1,128,1,float16,fp8,0,0.18493332465489706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,2,128,1,float16,float16,0,0.1776906649271647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,2,128,1,float16,fp8,0,0.19192000230153403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,4,128,1,float16,float16,0,0.18921599785486856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,4,128,1,float16,fp8,0,0.20973867177963257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,8,128,1,float16,float16,0,0.2037973403930664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,8,128,1,float16,fp8,0,0.2174933354059855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,32,128,1,float16,float16,0,0.127018670241038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,32,128,1,float16,fp8,0,0.1405333379904429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,1,128,1,float16,float16,0,0.09168533484141032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,1,128,1,float16,fp8,0,0.09921066959698994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,2,128,1,float16,float16,0,0.0976746678352356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,2,128,1,float16,fp8,0,0.1048426628112793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,4,128,1,float16,float16,0,0.10291733344395955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,4,128,1,float16,fp8,0,0.11214933792750041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,8,128,1,float16,float16,0,0.11568533380826314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,8,128,1,float16,fp8,0,0.12486400206883748
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,1,128,1,float16,float16,0,2.7613067626953125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,2,128,1,float16,float16,0,2.9355732599894204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,1,128,1,float16,fp8,0,3.009765307108561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,4,128,1,float16,float16,0,3.2629919052124023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,2,128,1,float16,fp8,0,3.1153386433919272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,4,128,1,float16,fp8,0,3.6411307652791343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,8,128,1,float16,float16,0,3.466517448425293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,8,128,1,float16,fp8,0,3.735322634379069
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,32,128,1,float16,float16,0,2.0971360206604004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,1,128,1,float16,float16,0,1.4104000727335613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,1,128,1,float16,fp8,0,1.5218453407287598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,32,128,1,float16,fp8,0,2.1831466356913247
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,2,128,1,float16,float16,0,1.4983359972635906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,4,128,1,float16,float16,0,1.6550079981486003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,2,128,1,float16,fp8,0,1.6101333300272624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,8,128,1,float16,float16,0,1.7434345881144206
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,4,128,1,float16,fp8,0,1.8729920387268066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,32,128,1,float16,float16,0,1.0398773352305095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,1,128,1,float16,float16,0,0.7089119752248129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,8,128,1,float16,fp8,0,1.8733065923055012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,32,128,1,float16,fp8,0,1.1174133618672688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,1,128,1,float16,fp8,0,0.7866933345794678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,2,128,1,float16,float16,0,0.775007963180542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,4,128,1,float16,float16,0,0.8427786827087402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,4,128,1,float16,fp8,0,0.9400959809621176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,8,128,1,float16,float16,0,0.8919786612192789
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,2,128,1,float16,fp8,0,0.8179946740468343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,8,128,1,float16,fp8,0,0.9609333674112955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,32,128,1,float16,float16,0,0.5231733322143555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,32,128,1,float16,fp8,0,0.5820000171661377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,1,128,1,float16,float16,0,0.3710613250732422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,1,128,1,float16,fp8,0,0.4063520034154256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,2,128,1,float16,float16,0,0.39762131373087567
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,2,128,1,float16,fp8,0,0.41870399316151935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,4,128,1,float16,float16,0,0.43861865997314453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,4,128,1,float16,fp8,0,0.4854400157928467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,8,128,1,float16,fp8,0,0.5071573257446289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,32,128,1,float16,float16,0,0.2816106677055359
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,32,128,1,float16,fp8,0,0.30593599875768024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,1,128,1,float16,float16,0,0.2003999948501587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,8,128,1,float16,float16,0,0.4665280183156331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,1,128,1,float16,fp8,0,0.21740265687306723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,2,128,1,float16,float16,0,0.21488000949223837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,2,128,1,float16,fp8,0,0.2246453364690145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,4,128,1,float16,float16,0,0.2310240070025126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,4,128,1,float16,fp8,0,0.26099199056625366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,8,128,1,float16,float16,0,0.24530667066574097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,8,128,1,float16,fp8,0,0.27007466554641724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,2,128,1,float16,float16,0,0.11773866415023804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,2,128,1,float16,fp8,0,0.12928533554077148
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,32,128,1,float16,float16,0,0.1548640032609304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,32,128,1,float16,fp8,0,0.16874667008717856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,1,128,1,float16,float16,0,0.11200533310572307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,1,128,1,float16,fp8,0,0.12296533584594727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,4,128,1,float16,float16,0,0.12746133406956991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,4,128,1,float16,fp8,0,0.13743999600410461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,8,128,1,float16,float16,0,0.13395733634630838
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,8,128,1,float16,fp8,0,0.15057067076365152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,32,128,1,float16,float16,0,0.08866666754086812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,32,128,1,float16,fp8,0,0.10064533352851868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,1,128,1,float16,float16,0,0.05991999804973602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,1,128,1,float16,fp8,0,0.06727999945481618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,2,128,1,float16,float16,0,0.06458666423956554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,2,128,1,float16,fp8,0,0.07172266642252605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,4,128,1,float16,float16,0,0.06890666484832764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,4,128,1,float16,fp8,0,0.07926400005817413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,8,128,1,float16,float16,0,0.07712000111738841
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,8,128,1,float16,fp8,0,0.08335999647776286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,1,128,1,float16,float16,0,2.9062185287475586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,1,128,1,float16,fp8,0,3.2161865234375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,2,128,1,float16,float16,0,3.129669189453125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,2,128,1,float16,fp8,0,3.3496694564819336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,8,128,1,float16,float16,0,3.7431519826253257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,4,128,1,float16,float16,0,3.5154345830281577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,8,128,1,float16,fp8,0,4.090986569722493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,32,128,1,float16,float16,0,2.2913173039754233
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,4,128,1,float16,fp8,0,4.030741373697917
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,1,128,1,float16,float16,0,1.4476745923360188
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,32,128,1,float16,fp8,0,2.537013371785482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,1,128,1,float16,fp8,0,1.6249225934346516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,2,128,1,float16,float16,0,1.6079999605814617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,32,128,1,float16,float16,0,1.1422026952107747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,4,128,1,float16,float16,0,1.7752799987792969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,32,128,1,float16,fp8,0,1.2848906517028809
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,1,128,1,float16,float16,0,0.7485386530558268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,8,128,1,float16,fp8,0,2.1055146853129068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,2,128,1,float16,fp8,0,1.6927839914957683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,4,128,1,float16,fp8,0,2.053775946299235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,8,128,1,float16,float16,0,1.8726080258687336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,1,128,1,float16,fp8,0,0.8250613212585449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,2,128,1,float16,float16,0,0.8126293023427328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,2,128,1,float16,fp8,0,0.859440008799235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,4,128,1,float16,float16,0,0.9131519794464111
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,4,128,1,float16,fp8,0,1.0342133045196533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,8,128,1,float16,float16,0,0.9820799827575684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,32,128,1,float16,float16,0,0.5861866474151611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,1,128,1,float16,float16,0,0.37881600856781006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,32,128,1,float16,fp8,0,0.667685349782308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,8,128,1,float16,fp8,0,1.0854132970174153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,1,128,1,float16,fp8,0,0.42612266540527344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,2,128,1,float16,float16,0,0.42420800526936847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,2,128,1,float16,fp8,0,0.45001065731048584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,4,128,1,float16,float16,0,0.4728800058364868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,8,128,1,float16,float16,0,0.49048535029093426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,4,128,1,float16,fp8,0,0.5294986565907797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,32,128,1,float16,float16,0,0.3059999942779541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,32,128,1,float16,fp8,0,0.34355731805165607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,8,128,1,float16,fp8,0,0.5603786706924438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,1,128,1,float16,float16,0,0.20535999536514282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,1,128,1,float16,fp8,0,0.22979732354482016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,2,128,1,float16,float16,0,0.22124266624450684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,2,128,1,float16,fp8,0,0.23803200324376425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,4,128,1,float16,float16,0,0.24952000379562378
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,4,128,1,float16,fp8,0,0.27748266855875653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,8,128,1,float16,float16,0,0.2649173339207967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,8,128,1,float16,fp8,0,0.2947733402252197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,32,128,1,float16,float16,0,0.16539200146993002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,32,128,1,float16,fp8,0,0.18684266010920206
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,1,128,1,float16,float16,0,0.11077866951624553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,1,128,1,float16,fp8,0,0.12821867068608603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,2,128,1,float16,float16,0,0.12309333682060242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,2,128,1,float16,fp8,0,0.13388267159461975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,4,128,1,float16,float16,0,0.13487999637921652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,8,128,1,float16,float16,0,0.14429333806037903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,4,128,1,float16,fp8,0,0.15414933363596597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,8,128,1,float16,fp8,0,0.1600106656551361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,32,128,1,float16,float16,0,0.09431999921798706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,2,128,1,float16,float16,0,0.0689386675755183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,32,128,1,float16,fp8,0,0.10786133011182149
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,1,128,1,float16,float16,0,0.0633493314186732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,1,128,1,float16,fp8,0,0.0719413310289383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,2,128,1,float16,fp8,0,0.07588266829649608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,4,128,1,float16,float16,0,0.06976533432801564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,8,128,1,float16,float16,0,0.08218133449554443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,4,128,1,float16,fp8,0,0.08042666812737782
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,1,128,1,float16,float16,0,0.03654933224121729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,8,128,1,float16,fp8,0,0.0912000040213267
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,32,128,1,float16,float16,0,0.056832000613212585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,32,128,1,float16,fp8,0,0.06649066507816315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,1,128,1,float16,fp8,0,0.0431573341290156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,2,128,1,float16,float16,0,0.041333332657814026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,2,128,1,float16,fp8,0,0.04654933512210846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,4,128,1,float16,float16,0,0.04419733087221781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,4,128,1,float16,fp8,0,0.05081599950790405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,8,128,1,float16,fp8,0,0.05170666674772898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,8,128,1,float16,float16,0,0.0450186679760615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,1,128,1,float16,float16,0,2.1678080558776855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,1,128,1,float16,fp8,0,2.47377602259318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,2,128,1,float16,fp8,0,2.621642589569092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,2,128,1,float16,float16,0,2.4034454027811685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,4,128,1,float16,float16,0,2.7699947357177734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,32,128,1,float16,float16,0,1.8947253227233887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,4,128,1,float16,fp8,0,3.2739785512288413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,32,128,1,float16,fp8,0,2.279797395070394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,1,128,1,float16,float16,0,1.0849706331888835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,8,128,1,float16,float16,0,3.012282689412435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,1,128,1,float16,fp8,0,1.232757329940796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,2,128,1,float16,fp8,0,1.319381316502889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,2,128,1,float16,float16,0,1.21451203028361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,4,128,1,float16,fp8,0,1.6490453084309895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,4,128,1,float16,float16,0,1.4080692927042644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,32,128,1,float16,float16,0,0.9603573481241862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,32,128,1,float16,fp8,0,1.1476960182189941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,1,128,1,float16,float16,0,0.5556319952011108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,8,128,1,float16,fp8,0,1.6902186075846355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,8,128,1,float16,float16,0,1.5078293482462566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,1,128,1,float16,fp8,0,0.6313866774241129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,2,128,1,float16,float16,0,0.6290293137232462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,2,128,1,float16,fp8,0,0.6734346548716227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,4,128,1,float16,fp8,0,0.8371413548787435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,4,128,1,float16,float16,0,0.716858704884847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,8,128,1,float16,float16,0,0.7797706921895345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,32,128,1,float16,float16,0,0.4906239906946818
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,8,128,1,float16,fp8,0,3.4138453801472983
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,32,128,1,float16,fp8,0,0.5842826763788859
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,8,128,1,float16,fp8,0,0.8892107009887695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,1,128,1,float16,float16,0,0.2901653250058492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,1,128,1,float16,fp8,0,0.32646934191385907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,2,128,1,float16,float16,0,0.3221973379453023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,2,128,1,float16,fp8,0,0.35024531682332355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,4,128,1,float16,float16,0,0.3712533315022786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,8,128,1,float16,float16,0,0.3998826742172241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,32,128,1,float16,float16,0,0.26126400629679364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,8,128,1,float16,fp8,0,0.4537546634674072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,32,128,1,float16,fp8,0,0.30510934193929035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,1,128,1,float16,float16,0,0.15505599975585938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,1,128,1,float16,fp8,0,0.17603733142217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,2,128,1,float16,float16,0,0.17056000232696533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,4,128,1,float16,fp8,0,0.4300533135732015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,2,128,1,float16,fp8,0,0.18787733713785806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,4,128,1,float16,float16,0,0.19819200038909912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,4,128,1,float16,fp8,0,0.2246239980061849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,8,128,1,float16,float16,0,0.2100693384806315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,8,128,1,float16,fp8,0,0.24269866943359375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,2,128,1,float16,float16,0,0.09547733267148335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,32,128,1,float16,float16,0,0.1383680005868276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,32,128,1,float16,fp8,0,0.16876266400019327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,1,128,1,float16,float16,0,0.0872213343779246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,1,128,1,float16,fp8,0,0.10051733255386353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,2,128,1,float16,fp8,0,0.10631466905275981
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,4,128,1,float16,float16,0,0.10616000493367513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,4,128,1,float16,fp8,0,0.12355200449625652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,8,128,1,float16,float16,0,0.11506133278210957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,32,128,1,float16,float16,0,0.07865599791208903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,8,128,1,float16,fp8,0,0.1327893336613973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,32,128,1,float16,fp8,0,0.09533333778381348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,1,128,1,float16,float16,0,0.049423997600873314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,1,128,1,float16,fp8,0,0.05632533133029938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,2,128,1,float16,float16,0,0.05375466744105021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,2,128,1,float16,fp8,0,0.05762133498986562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,4,128,1,float16,float16,0,0.055434669057528176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,4,128,1,float16,fp8,0,0.063509335120519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,8,128,1,float16,float16,0,0.06700799862543742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,8,128,1,float16,fp8,0,0.07551999886830647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,1,128,1,float16,fp8,0,0.035061334570248924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,32,128,1,float16,float16,0,0.046282668908437095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,32,128,1,float16,fp8,0,0.05412800113360087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,1,128,1,float16,float16,0,0.030218665798505146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,2,128,1,float16,float16,0,0.031061333914597828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,2,128,1,float16,fp8,0,0.036330667634805046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,4,128,1,float16,float16,0,0.033626665671666466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,4,128,1,float16,fp8,0,0.038912000755469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,8,128,1,float16,fp8,0,0.03916800022125244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,8,128,1,float16,float16,0,0.03485333422819773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,32,128,1,float16,float16,0,0.02327466756105423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,32,128,1,float16,fp8,0,0.028901333610216778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,1,128,1,float16,float16,0,0.020047999918460846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,1,128,1,float16,fp8,0,0.023317334552605946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,2,128,1,float16,float16,0,0.019946667055288952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,2,128,1,float16,fp8,0,0.023578666150569916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,4,128,1,float16,float16,0,0.02165866643190384
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,8,128,1,float16,float16,0,0.021498667697111767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,4,128,1,float16,fp8,0,0.025263999899228413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,8,128,1,float16,fp8,0,0.02566933383544286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,1,128,1,float16,float16,0,0.9059893290201823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,1,128,1,float16,fp8,0,1.0360906918843586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,2,128,1,float16,float16,0,1.0343786875406902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,4,128,1,float16,float16,0,1.2325546741485596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,2,128,1,float16,fp8,0,1.1346186796824138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,8,128,1,float16,float16,0,1.330298662185669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,32,128,1,float16,float16,0,0.8836212952931722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,32,128,1,float16,fp8,0,0.9920106728871664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,1,128,1,float16,fp8,0,0.5293706655502319
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,2,128,1,float16,float16,0,0.5367466608683268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,8,128,1,float16,fp8,0,1.5014026959737141
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,1,128,1,float16,float16,0,0.467578649520874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,4,128,1,float16,fp8,0,1.4617013931274414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,4,128,1,float16,fp8,0,0.7357707023620605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,4,128,1,float16,float16,0,0.630672017733256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,32,128,1,float16,fp8,0,0.5121013323465983
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,8,128,1,float16,float16,0,0.6881013711293539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,1,128,1,float16,float16,0,0.24347732464472452
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,8,128,1,float16,fp8,0,0.7996853192647299
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,2,128,1,float16,float16,0,0.27511467536290485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,1,128,1,float16,fp8,0,0.27640000979105633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,2,128,1,float16,fp8,0,0.3044480085372925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,2,128,1,float16,fp8,0,0.5786293347676595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,4,128,1,float16,fp8,0,0.383568008740743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,32,128,1,float16,float16,0,0.45240533351898193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,4,128,1,float16,float16,0,0.32181332508722943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,32,128,1,float16,float16,0,0.23498133818308511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,8,128,1,float16,float16,0,0.3553333282470703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,1,128,1,float16,float16,0,0.13199466466903687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,32,128,1,float16,fp8,0,0.2717706759770711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,4,128,1,float16,float16,0,0.1699840029080709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,1,128,1,float16,fp8,0,0.15093866984049478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,2,128,1,float16,float16,0,0.14797332882881165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,8,128,1,float16,fp8,0,0.4095733165740967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,2,128,1,float16,fp8,0,0.16400532921155295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,4,128,1,float16,fp8,0,0.20139199495315552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,8,128,1,float16,float16,0,0.18816532691319784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,8,128,1,float16,fp8,0,0.21716266870498657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,32,128,1,float16,float16,0,0.12797333796819052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,32,128,1,float16,fp8,0,0.14678399761517844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,1,128,1,float16,float16,0,0.07424533367156982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,1,128,1,float16,fp8,0,0.0862613320350647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,2,128,1,float16,float16,0,0.08227199812730153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,2,128,1,float16,fp8,0,0.09406933188438416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,4,128,1,float16,float16,0,0.09300800164540608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,4,128,1,float16,fp8,0,0.1111946702003479
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,8,128,1,float16,float16,0,0.10269332925478618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,8,128,1,float16,fp8,0,0.12070932984352112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,32,128,1,float16,float16,0,0.0748533308506012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,32,128,1,float16,fp8,0,0.08547199765841167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,1,128,1,float16,float16,0,0.04111466556787491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,1,128,1,float16,fp8,0,0.049216002225875854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,2,128,1,float16,float16,0,0.045738667249679565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,2,128,1,float16,fp8,0,0.04997866849104563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,4,128,1,float16,float16,0,0.046181331078211464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,4,128,1,float16,fp8,0,0.05420800050099691
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,8,128,1,float16,float16,0,0.058335999647776283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,8,128,1,float16,fp8,0,0.06680533289909363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,32,128,1,float16,float16,0,0.0406986673672994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,32,128,1,float16,fp8,0,0.047168001532554626
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,4,128,1,float16,float16,0,0.028016000986099243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,1,128,1,float16,float16,0,0.025034666061401367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,1,128,1,float16,fp8,0,0.029706666866938274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,2,128,1,float16,float16,0,0.025663999219735462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,2,128,1,float16,fp8,0,0.03012799968322118
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,4,128,1,float16,fp8,0,0.03305600086847941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,8,128,1,float16,float16,0,0.028192001084486645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,8,128,1,float16,fp8,0,0.034058667719364166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,32,128,1,float16,float16,0,0.02046400060256322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,32,128,1,float16,fp8,0,0.02585600068171819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,1,128,1,float16,float16,0,0.017301333447297413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,1,128,1,float16,fp8,0,0.019786667078733444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,2,128,1,float16,float16,0,0.017429333180189133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,2,128,1,float16,fp8,0,0.019978666057189304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,4,128,1,float16,float16,0,0.018853332847356796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,4,128,1,float16,fp8,0,0.021568000316619873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,8,128,1,float16,float16,0,0.018986667195955913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,8,128,1,float16,fp8,0,0.022133332987626392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,32,128,1,float16,float16,0,0.017632000148296356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,32,128,1,float16,fp8,0,0.022469334304332733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,1,128,1,float16,float16,0,0.016522667060295742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,1,128,1,float16,fp8,0,0.01966933285196622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,2,128,1,float16,float16,0,0.016656000167131424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,2,128,1,float16,fp8,0,0.019530666371186573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,4,128,1,float16,float16,0,0.01728533332546552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,4,128,1,float16,fp8,0,0.019776000330845516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,8,128,1,float16,float16,0,0.01700266698996226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,8,128,1,float16,fp8,0,0.020069333414236706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,1,128,1,float16,float16,0,0.4410986502965291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,1,128,1,float16,fp8,0,0.47439467906951904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,2,128,1,float16,float16,0,0.5025333166122437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,2,128,1,float16,fp8,0,0.5267200072606405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,4,128,1,float16,float16,0,0.6273173491160074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,4,128,1,float16,fp8,0,0.6583573420842489
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,32,128,1,float16,float16,0,0.43294934431711835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,8,128,1,float16,fp8,0,0.7506293455759684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,8,128,1,float16,float16,0,0.6648906469345093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,1,128,1,float16,float16,0,0.23194666703542074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,32,128,1,float16,fp8,0,0.4553866783777873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,1,128,1,float16,fp8,0,0.25643734137217206
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,2,128,1,float16,float16,0,0.26812267303466797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,8,128,1,float16,float16,0,0.3404800097147624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,32,128,1,float16,fp8,0,0.24159467220306396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,2,128,1,float16,fp8,0,0.27640000979105633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,4,128,1,float16,float16,0,0.32073066631952923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,1,128,1,float16,float16,0,0.1255466639995575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,4,128,1,float16,fp8,0,0.342303991317749
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,8,128,1,float16,fp8,0,0.3909173409144084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,1,128,1,float16,fp8,0,0.14230400323867798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,2,128,1,float16,fp8,0,0.15225600202878317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,4,128,1,float16,fp8,0,0.18411733706792197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,2,128,1,float16,float16,0,0.14147733648618063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,4,128,1,float16,float16,0,0.16638933618863425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,32,128,1,float16,float16,0,0.2278719941775004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,8,128,1,float16,float16,0,0.17871999740600586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,32,128,1,float16,float16,0,0.12310399611790974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,32,128,1,float16,fp8,0,0.1320693294207255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,8,128,1,float16,fp8,0,0.21033066511154175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,1,128,1,float16,float16,0,0.07243200143178304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,1,128,1,float16,fp8,0,0.08172800143559773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,2,128,1,float16,float16,0,0.07984533409277599
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,4,128,1,float16,float16,0,0.08729599912961324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,2,128,1,float16,fp8,0,0.08799466490745544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,8,128,1,float16,float16,0,0.09904000163078308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,4,128,1,float16,fp8,0,0.10215999682744344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,32,128,1,float16,float16,0,0.07090666890144348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,8,128,1,float16,fp8,0,0.11462933818499248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,32,128,1,float16,fp8,0,0.07522133489449818
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,1,128,1,float16,float16,0,0.03668266783157984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,1,128,1,float16,fp8,0,0.043151999513308205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,8,128,1,float16,float16,0,0.05663999915122986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,2,128,1,float16,float16,0,0.0413973331451416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,2,128,1,float16,fp8,0,0.045834665497144066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,4,128,1,float16,float16,0,0.04195733368396759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,4,128,1,float16,fp8,0,0.048122664292653404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,8,128,1,float16,fp8,0,0.06113600234190623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,32,128,1,float16,float16,0,0.03993066648642222
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,32,128,1,float16,fp8,0,0.0412266676624616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,1,128,1,float16,float16,0,0.022410665949185688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,1,128,1,float16,fp8,0,0.02516799916823705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,2,128,1,float16,float16,0,0.02271466702222824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,2,128,1,float16,fp8,0,0.02647999922434489
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,4,128,1,float16,float16,0,0.025722667574882507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,4,128,1,float16,fp8,0,0.02921066681543986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,8,128,1,float16,float16,0,0.02589866767326991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,8,128,1,float16,fp8,0,0.029877332349618275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,2,128,1,float16,float16,0,0.016127999871969223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,2,128,1,float16,fp8,0,0.01830400029818217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,32,128,1,float16,float16,0,0.019018666197856266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,4,128,1,float16,float16,0,0.017242666333913803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,32,128,1,float16,fp8,0,0.02170666555563609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,1,128,1,float16,float16,0,0.015781333049138386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,32,128,1,float16,float16,0,0.016666666915019352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,32,128,1,float16,fp8,0,0.018826667219400406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,1,128,1,float16,fp8,0,0.018346666047970455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,4,128,1,float16,fp8,0,0.019589333484570186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,8,128,1,float16,float16,0,0.017498667041460674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,8,128,1,float16,fp8,0,0.02004266654451688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,1,128,1,float16,float16,0,0.015279999623696009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,1,128,1,float16,fp8,0,0.018042666216691334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,2,128,1,float16,float16,0,0.01563199982047081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,2,128,1,float16,fp8,0,0.018016000588734944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,32,128,1,float16,float16,0,0.015957333147525787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,4,128,1,float16,float16,0,0.01590399940808614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,1,128,1,float16,float16,0,0.015360000232855478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,4,128,1,float16,fp8,0,0.01766933376590411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,2,128,1,float16,float16,0,0.014970666418472925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,8,128,1,float16,float16,0,0.015605332950750986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,8,128,1,float16,fp8,0,0.017759999881188076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,32,128,1,float16,fp8,0,0.017973333597183228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,1,128,1,float16,fp8,0,0.01793066660563151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,2,128,1,float16,fp8,0,0.017610666652520496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,4,128,1,float16,float16,0,0.015210667004187902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,4,128,1,float16,fp8,0,0.017136000096797943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,8,128,1,float16,float16,0,0.015087999403476715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,8,128,1,float16,fp8,0,0.01746133342385292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,1,128,1,float16,float16,0,0.2307093342145284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,1,128,1,float16,fp8,0,0.25672000646591187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,2,128,1,float16,float16,0,0.2675679922103882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,2,128,1,float16,fp8,0,0.2768053412437439
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,4,128,1,float16,float16,0,0.31782400608062744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,32,128,1,float16,float16,0,0.2495786746342977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,8,128,1,float16,fp8,0,0.3852693239847819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,32,128,1,float16,fp8,0,0.2391093373298645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,4,128,1,float16,fp8,0,0.3409706751505534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,8,128,1,float16,float16,0,0.3407306671142578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,1,128,1,float16,float16,0,0.1246560017267863
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,1,128,1,float16,fp8,0,0.1386133333047231
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,8,128,1,float16,fp8,0,0.20855466524759927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,32,128,1,float16,float16,0,0.13335999846458435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,2,128,1,float16,fp8,0,0.153029332558314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,2,128,1,float16,float16,0,0.1437386671702067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,4,128,1,float16,float16,0,0.16588800152142844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,4,128,1,float16,fp8,0,0.18210667371749878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,8,128,1,float16,float16,0,0.178874671459198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,32,128,1,float16,fp8,0,0.1274720033009847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,1,128,1,float16,fp8,0,0.08167999982833862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,1,128,1,float16,float16,0,0.07202133536338806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,2,128,1,float16,float16,0,0.0807360013326009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,2,128,1,float16,fp8,0,0.08743466933568318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,4,128,1,float16,fp8,0,0.09981333216031392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,8,128,1,float16,float16,0,0.09914666414260864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,4,128,1,float16,float16,0,0.08802133798599243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,32,128,1,float16,float16,0,0.07493333518505096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,8,128,1,float16,fp8,0,0.11493866642316182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,32,128,1,float16,fp8,0,0.07220800220966339
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,1,128,1,float16,float16,0,0.03598399957021078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,1,128,1,float16,fp8,0,0.04030400017897288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,2,128,1,float16,float16,0,0.04125333329041799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,2,128,1,float16,fp8,0,0.04340800146261851
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,4,128,1,float16,float16,0,0.04200533529122671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,32,128,1,float16,fp8,0,0.03736000011364619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,4,128,1,float16,fp8,0,0.04825599988301595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,8,128,1,float16,float16,0,0.055914665261904396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,8,128,1,float16,fp8,0,0.06057600180308024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,32,128,1,float16,float16,0,0.04297600189844767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,1,128,1,float16,float16,0,0.02165333429972331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,1,128,1,float16,fp8,0,0.0249493345618248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,2,128,1,float16,float16,0,0.022869333624839783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,4,128,1,float16,float16,0,0.02531733363866806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,2,128,1,float16,fp8,0,0.0262719988822937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,4,128,1,float16,fp8,0,0.028981332977612812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,8,128,1,float16,float16,0,0.025402667621771496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,8,128,1,float16,fp8,0,0.029989334444204967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,1,128,1,float16,fp8,0,0.018677332748969395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,32,128,1,float16,float16,0,0.02032533288002014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,32,128,1,float16,fp8,0,0.02053333322207133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,1,128,1,float16,float16,0,0.01594666639963786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,2,128,1,float16,float16,0,0.01591466615597407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,2,128,1,float16,fp8,0,0.018405333161354065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,4,128,1,float16,float16,0,0.017407999684413273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,4,128,1,float16,fp8,0,0.01971199984351794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,8,128,1,float16,float16,0,0.01730666682124138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,8,128,1,float16,fp8,0,0.020234666764736176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,32,128,1,float16,float16,0,0.014453332871198654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,2,128,1,float16,fp8,0,0.017935999979575474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,32,128,1,float16,fp8,0,0.014773332824309668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,1,128,1,float16,float16,0,0.015344000111023584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,1,128,1,float16,fp8,0,0.0180479995906353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,2,128,1,float16,float16,0,0.015109332899252573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,4,128,1,float16,float16,0,0.015392000476519266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,4,128,1,float16,fp8,0,0.01807466646035512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,8,128,1,float16,float16,0,0.015856000284353893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,8,128,1,float16,fp8,0,0.0182239996890227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,32,128,1,float16,float16,0,0.013471999516089758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,32,128,1,float16,fp8,0,0.014085333794355392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,1,128,1,float16,float16,0,0.015370666980743408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,1,128,1,float16,fp8,0,0.017765333255132038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,2,128,1,float16,float16,0,0.015066667149464289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,2,128,1,float16,fp8,0,0.018005333840847015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,4,128,1,float16,float16,0,0.014837333311637243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,4,128,1,float16,fp8,0,0.01711999997496605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,8,128,1,float16,float16,0,0.015200000256299973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,8,128,1,float16,fp8,0,0.01757866640885671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,32,128,1,float16,fp8,0,0.013861333330472311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,32,128,1,float16,float16,0,0.013349333157142004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,1,128,1,float16,float16,0,0.014352000008026758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,1,128,1,float16,fp8,0,0.016554666062196095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,2,128,1,float16,float16,0,0.014394666999578476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,2,128,1,float16,fp8,0,0.01632000009218852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,4,128,1,float16,float16,0,0.014511999984582266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,4,128,1,float16,fp8,0,0.01611199975013733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,8,128,1,float16,float16,0,0.01451733335852623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,8,128,1,float16,fp8,0,0.0173333336909612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,1,128,1,float16,float16,0,0.12409067153930664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,1,128,1,float16,fp8,0,0.14219733079274496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,2,128,1,float16,float16,0,0.14069867134094238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,2,128,1,float16,fp8,0,0.15271466970443726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,4,128,1,float16,float16,0,0.16378666957219443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,32,128,1,float16,float16,0,0.18177066246668497
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,4,128,1,float16,fp8,0,0.18222934007644653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,32,128,1,float16,fp8,0,0.17626667022705078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,1,128,1,float16,float16,0,0.07212266822655995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,8,128,1,float16,float16,0,0.17683200041453043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,8,128,1,float16,fp8,0,0.20894400278727213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,1,128,1,float16,fp8,0,0.0813920001188914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,2,128,1,float16,fp8,0,0.08773866295814514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,2,128,1,float16,float16,0,0.08000533282756805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,4,128,1,float16,float16,0,0.08571199576059978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,4,128,1,float16,fp8,0,0.10110933581988017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,8,128,1,float16,float16,0,0.09826667110125224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,32,128,1,float16,float16,0,0.09771200021107991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,32,128,1,float16,fp8,0,0.09585600097974141
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,8,128,1,float16,fp8,0,0.1141813298066457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,1,128,1,float16,float16,0,0.03716800113519033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,1,128,1,float16,fp8,0,0.04010133445262909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,2,128,1,float16,float16,0,0.04098666707674662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,2,128,1,float16,fp8,0,0.04267199834187826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,8,128,1,float16,float16,0,0.05618133147557577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,4,128,1,float16,float16,0,0.042170668641726174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,4,128,1,float16,fp8,0,0.04841599861780802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,32,128,1,float16,float16,0,0.054789334535598755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,8,128,1,float16,fp8,0,0.06249066690603892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,32,128,1,float16,fp8,0,0.048581331968307495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,1,128,1,float16,float16,0,0.02141333371400833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,1,128,1,float16,fp8,0,0.024890666206677754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,2,128,1,float16,float16,0,0.02271466702222824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,2,128,1,float16,fp8,0,0.026213333010673523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,4,128,1,float16,float16,0,0.02521066615978877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,4,128,1,float16,fp8,0,0.029194665451844532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,8,128,1,float16,float16,0,0.025834667185942333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,8,128,1,float16,fp8,0,0.029535998900731403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,32,128,1,float16,float16,0,0.026922665536403656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,32,128,1,float16,fp8,0,0.027072000006834667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,1,128,1,float16,float16,0,0.01562133307258288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,1,128,1,float16,fp8,0,0.01852799952030182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,2,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,2,128,1,float16,fp8,0,0.018133333573738735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,4,128,1,float16,float16,0,0.017263999829689663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,4,128,1,float16,fp8,0,0.020010666300853092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,8,128,1,float16,float16,0,0.017210666090250015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,8,128,1,float16,fp8,0,0.019930666933457058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,32,128,1,float16,float16,0,0.018085333208243053
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,32,128,1,float16,fp8,0,0.018383999665578205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,1,128,1,float16,float16,0,0.015306666493415833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,1,128,1,float16,fp8,0,0.01752000053723653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,2,128,1,float16,float16,0,0.01573333392540614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,2,128,1,float16,fp8,0,0.01757866640885671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,8,128,1,float16,fp8,0,0.01766933376590411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,4,128,1,float16,float16,0,0.015658666690190632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,8,128,1,float16,float16,0,0.015498666713635126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,4,128,1,float16,fp8,0,0.017973333597183228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,32,128,1,float16,float16,0,0.013573333621025085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,1,128,1,float16,float16,0,0.015013333410024643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,32,128,1,float16,fp8,0,0.014442666123310724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,2,128,1,float16,float16,0,0.014981333166360855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,2,128,1,float16,fp8,0,0.01747200017174085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,1,128,1,float16,fp8,0,0.01773333301146825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,4,128,1,float16,float16,0,0.015050667027632395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,4,128,1,float16,fp8,0,0.01727466657757759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,8,128,1,float16,float16,0,0.014826666563749313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,8,128,1,float16,fp8,0,0.017466666797796886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,32,128,1,float16,float16,0,0.01351999988158544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,32,128,1,float16,fp8,0,0.013647999614477158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,1,128,1,float16,float16,0,0.014218666901191076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,1,128,1,float16,fp8,0,0.01655999943614006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,2,128,1,float16,float16,0,0.014384000251690546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,2,128,1,float16,fp8,0,0.01646399994691213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,4,128,1,float16,float16,0,0.014592000593741735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,4,128,1,float16,fp8,0,0.017162666966517765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,8,128,1,float16,float16,0,0.014495999862750372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,8,128,1,float16,fp8,0,0.016309333344300587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,32,128,1,float16,float16,0,0.013194666554530462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,32,128,1,float16,fp8,0,0.013631999492645264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,1,128,1,float16,float16,0,0.014229333649079004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,2,128,1,float16,float16,0,0.014101333916187286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,1,128,1,float16,fp8,0,0.0163680004576842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,2,128,1,float16,fp8,0,0.01617066686352094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,4,128,1,float16,float16,0,0.014325333138306936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,4,128,1,float16,fp8,0,0.016074666132529575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,8,128,1,float16,float16,0,0.01414399966597557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,8,128,1,float16,fp8,0,0.0161013330022494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,1,128,1,float16,float16,0,0.0713973343372345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,1,128,1,float16,fp8,0,0.08097599943478902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,2,128,1,float16,float16,0,0.07925866544246674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,2,128,1,float16,fp8,0,0.08691199620564778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,4,128,1,float16,float16,0,0.08629332979520161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,4,128,1,float16,fp8,0,0.10058666268984477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,32,128,1,float16,float16,0,0.14941866199175516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,8,128,1,float16,float16,0,0.11005333065986633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,32,128,1,float16,fp8,0,0.142767995595932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,8,128,1,float16,fp8,0,0.10852799812952678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,1,128,1,float16,fp8,0,0.04041599979003271
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,1,128,1,float16,float16,0,0.03825599948565165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,2,128,1,float16,float16,0,0.04248000184694926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,2,128,1,float16,fp8,0,0.04246933261553446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,4,128,1,float16,fp8,0,0.047653332352638245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,4,128,1,float16,float16,0,0.041365332901477814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,8,128,1,float16,float16,0,0.06284800171852112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,32,128,1,float16,float16,0,0.07973866661389668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,1,128,1,float16,fp8,0,0.02477866659561793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,32,128,1,float16,fp8,0,0.0747680018345515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,8,128,1,float16,fp8,0,0.05629333357016245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,1,128,1,float16,float16,0,0.02184533327817917
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,2,128,1,float16,float16,0,0.022613334159056347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,2,128,1,float16,fp8,0,0.026127999027570088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,4,128,1,float16,float16,0,0.02499199906984965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,4,128,1,float16,fp8,0,0.028853334486484528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,8,128,1,float16,float16,0,0.028223998844623566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,8,128,1,float16,fp8,0,0.02951466788848241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,32,128,1,float16,float16,0,0.03950933367013931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,32,128,1,float16,fp8,0,0.03880000114440918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,1,128,1,float16,float16,0,0.015674666812022526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,1,128,1,float16,fp8,0,0.01825599993268649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,2,128,1,float16,float16,0,0.016021333634853363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,8,128,1,float16,float16,0,0.01855466639002164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,2,128,1,float16,fp8,0,0.018415999909241993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,4,128,1,float16,float16,0,0.016832000265518825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,4,128,1,float16,fp8,0,0.019573333362738293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,8,128,1,float16,fp8,0,0.019215999792019527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,32,128,1,float16,float16,0,0.024186665813128155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,32,128,1,float16,fp8,0,0.024288001159826916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,1,128,1,float16,float16,0,0.015263999501864115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,1,128,1,float16,fp8,0,0.017946666727463405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,2,128,1,float16,float16,0,0.01534933348496755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,2,128,1,float16,fp8,0,0.018063999712467194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,4,128,1,float16,float16,0,0.015781333049138386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,8,128,1,float16,float16,0,0.01358933374285698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,4,128,1,float16,fp8,0,0.01743999992807706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,1,128,1,float16,float16,0,0.01552533358335495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,1,128,1,float16,fp8,0,0.017887999614079792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,8,128,1,float16,fp8,0,0.013850666582584381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,32,128,1,float16,float16,0,0.01720533271630605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,32,128,1,float16,fp8,0,0.017653333644072216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,2,128,1,float16,float16,0,0.015008000036080679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,2,128,1,float16,fp8,0,0.01739199956258138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,4,128,1,float16,float16,0,0.015568000574906668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,4,128,1,float16,fp8,0,0.017114666601022083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,8,128,1,float16,float16,0,0.013445333888133367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,8,128,1,float16,fp8,0,0.013744000345468521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,32,128,1,float16,float16,0,0.013536000003417334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,32,128,1,float16,fp8,0,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,1,128,1,float16,float16,0,0.014815999815861383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,1,128,1,float16,fp8,0,0.016469333320856094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,2,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,2,128,1,float16,fp8,0,0.01640533283352852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,4,128,1,float16,float16,0,0.014202666779359182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,4,128,1,float16,fp8,0,0.0162773331006368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,8,128,1,float16,float16,0,0.012938667088747025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,8,128,1,float16,fp8,0,0.013610667238632837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,32,128,1,float16,float16,0,0.013253333667914072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,32,128,1,float16,fp8,0,0.01403733342885971
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,1,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,1,128,1,float16,fp8,0,0.016063999384641647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,2,128,1,float16,float16,0,0.014261333892742792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,2,128,1,float16,fp8,0,0.016122666498025257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,4,128,1,float16,float16,0,0.013951999445756277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,4,128,1,float16,fp8,0,0.015834666788578033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,8,128,1,float16,float16,0,0.012991999586423239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,8,128,1,float16,fp8,0,0.013269333789745966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,32,128,1,float16,float16,0,0.013349333157142004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,32,128,1,float16,fp8,0,0.013728000223636627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,1,128,1,float16,float16,0,0.013914667069911957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,1,128,1,float16,fp8,0,0.015978666643301647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,2,128,1,float16,float16,0,0.013839999834696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,2,128,1,float16,fp8,0,0.015978666643301647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,4,128,1,float16,float16,0,0.013562666873137156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,4,128,1,float16,fp8,0,0.01543466622630755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,8,128,1,float16,float16,0,0.01219733307758967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,8,128,1,float16,fp8,0,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,1,128,1,float16,fp8,0,20.529647827148438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,1,128,1,float16,float16,0,22.567209879557293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,2,128,1,float16,fp8,0,20.675268809000652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,4,128,1,float16,fp8,0,21.419199625651043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,2,128,1,float16,float16,0,23.327471415201824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,8,128,1,float16,fp8,0,21.441126505533855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,4,128,1,float16,float16,0,23.528485616048176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,8,128,1,float16,float16,0,23.335233052571613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,24,128,1,float16,float16,0,12.350746154785156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,1,128,1,float16,float16,0,11.832303365071615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,1,128,1,float16,fp8,0,10.30793571472168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,24,128,1,float16,fp8,0,11.099168141682943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,2,128,1,float16,fp8,0,10.407279968261719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,2,128,1,float16,float16,0,11.7881711324056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,4,128,1,float16,float16,0,12.41583506266276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,4,128,1,float16,fp8,0,10.815221150716146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,24,128,1,float16,fp8,0,5.585626602172852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,24,128,1,float16,float16,0,6.153818766276042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,1,128,1,float16,float16,0,5.841546376546224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,1,128,1,float16,fp8,0,5.276634534200032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,2,128,1,float16,float16,0,5.588725407918294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,2,128,1,float16,fp8,0,5.246218681335449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,8,128,1,float16,fp8,0,10.835253397623697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,8,128,1,float16,float16,0,11.502991994222006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,24,128,1,float16,float16,0,2.9408480326334634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,24,128,1,float16,fp8,0,2.8475093841552734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,4,128,1,float16,float16,0,5.617861429850261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,4,128,1,float16,fp8,0,5.712826410929362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,8,128,1,float16,float16,0,6.018213272094727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,8,128,1,float16,fp8,0,5.508517583211263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,1,128,1,float16,float16,0,2.685765266418457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,2,128,1,float16,float16,0,2.8592427571614585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,1,128,1,float16,fp8,0,2.989583969116211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,2,128,1,float16,fp8,0,2.66265598932902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,4,128,1,float16,float16,0,2.8683039347330728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,4,128,1,float16,fp8,0,3.0896905263264975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,8,128,1,float16,float16,0,2.9189332326253257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,8,128,1,float16,fp8,0,2.7967519760131836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,1,128,1,float16,fp8,0,11.788106282552084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,1,128,1,float16,float16,0,13.51907730102539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,2,128,1,float16,fp8,0,11.85091781616211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,2,128,1,float16,float16,0,13.788256327311197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,4,128,1,float16,fp8,0,12.69052251180013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,4,128,1,float16,float16,0,14.000559488932291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,8,128,1,float16,float16,0,13.885093688964844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,1,128,1,float16,float16,0,6.255365371704102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,24,128,1,float16,float16,0,7.009904225667317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,24,128,1,float16,fp8,0,6.490245183308919
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,2,128,1,float16,fp8,0,6.054298400878906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,2,128,1,float16,float16,0,7.228261311848958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,1,128,1,float16,fp8,0,5.973077138264974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,4,128,1,float16,float16,0,6.599557240804036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,8,128,1,float16,fp8,0,12.9431521097819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,24,128,1,float16,float16,0,3.452650705973307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,24,128,1,float16,fp8,0,3.6748425165812173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,1,128,1,float16,float16,0,3.0384534200032554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,8,128,1,float16,fp8,0,6.3107039133707685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,4,128,1,float16,fp8,0,6.742810567220052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,1,128,1,float16,fp8,0,3.249600092569987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,8,128,1,float16,float16,0,7.744112014770508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,2,128,1,float16,float16,0,3.186645189921061
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,2,128,1,float16,fp8,0,3.4782241185506186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,24,128,1,float16,float16,0,1.7193279266357422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,4,128,1,float16,float16,0,3.2862720489501953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,24,128,1,float16,fp8,0,1.898255983988444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,1,128,1,float16,float16,0,1.6098507245381672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,8,128,1,float16,float16,0,3.189759890238444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,4,128,1,float16,fp8,0,3.700064023335775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,8,128,1,float16,fp8,0,3.221280097961426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,1,128,1,float16,fp8,0,1.655471960703532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,2,128,1,float16,float16,0,1.5936853090922039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,2,128,1,float16,fp8,0,1.6981919606526692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,4,128,1,float16,float16,0,1.6831787427266438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,4,128,1,float16,fp8,0,1.846384048461914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,8,128,1,float16,float16,0,1.6528479258219402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,8,128,1,float16,fp8,0,1.758639971415202
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,1,128,1,float16,fp8,0,8.478901545206705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,1,128,1,float16,float16,0,9.349418640136719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,2,128,1,float16,fp8,0,8.497029622395834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,2,128,1,float16,float16,0,9.617194493611654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,4,128,1,float16,float16,0,10.068901062011719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,4,128,1,float16,fp8,0,9.003802617390951
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,8,128,1,float16,float16,0,9.815306981404623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,8,128,1,float16,fp8,0,8.954303741455078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,1,128,1,float16,float16,0,4.598688125610352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,24,128,1,float16,float16,0,4.912746747334798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,24,128,1,float16,fp8,0,4.7870025634765625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,1,128,1,float16,fp8,0,4.708079973856608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,2,128,1,float16,fp8,0,4.255322774251302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,2,128,1,float16,float16,0,4.551466623942058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,4,128,1,float16,float16,0,4.574399948120117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,4,128,1,float16,fp8,0,4.482346534729004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,24,128,1,float16,float16,0,2.368954658508301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,24,128,1,float16,fp8,0,2.399386723836263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,1,128,1,float16,float16,0,2.1281332969665527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,1,128,1,float16,fp8,0,2.2303147315979004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,2,128,1,float16,float16,0,2.1912585894266763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,8,128,1,float16,float16,0,4.939344088236491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,2,128,1,float16,fp8,0,2.157477378845215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,8,128,1,float16,fp8,0,4.993050575256348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,4,128,1,float16,float16,0,2.2813332875569663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,4,128,1,float16,fp8,0,2.3013973236083984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,24,128,1,float16,float16,0,1.2421279748280842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,8,128,1,float16,float16,0,2.303098678588867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,8,128,1,float16,fp8,0,2.317392031351725
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,1,128,1,float16,fp8,0,1.1178452968597412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,24,128,1,float16,fp8,0,1.334330717722575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,1,128,1,float16,float16,0,1.0986773173014324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,2,128,1,float16,float16,0,1.126757303873698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,2,128,1,float16,fp8,0,1.1354080041249592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,4,128,1,float16,float16,0,1.1696320374806721
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,4,128,1,float16,fp8,0,1.27564803759257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,8,128,1,float16,float16,0,1.1863946914672852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,8,128,1,float16,fp8,0,1.2060693105061848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,1,128,1,float16,fp8,0,10.854991912841797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,1,128,1,float16,float16,0,12.724100748697916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,2,128,1,float16,fp8,0,11.013956705729166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,2,128,1,float16,float16,0,12.320869445800781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,4,128,1,float16,float16,0,12.673759460449219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,4,128,1,float16,fp8,0,11.92349370320638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,8,128,1,float16,fp8,0,11.756059010823568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,8,128,1,float16,float16,0,13.058464050292969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,1,128,1,float16,float16,0,6.059216181437175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,24,128,1,float16,float16,0,6.559034983317058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,1,128,1,float16,fp8,0,5.72755750020345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,24,128,1,float16,fp8,0,6.251525243123372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,2,128,1,float16,fp8,0,5.5229441324869795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,2,128,1,float16,float16,0,5.862288157145183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,4,128,1,float16,float16,0,6.2068532307942705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,4,128,1,float16,fp8,0,5.980752309163411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,24,128,1,float16,float16,0,3.111882527669271
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,1,128,1,float16,float16,0,2.806960105895996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,24,128,1,float16,fp8,0,3.148922602335612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,1,128,1,float16,fp8,0,3.044981320699056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,2,128,1,float16,float16,0,2.8553441365559897
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,8,128,1,float16,float16,0,6.374325434366862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,8,128,1,float16,fp8,0,6.176191965738933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,2,128,1,float16,fp8,0,3.019359906514486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,4,128,1,float16,float16,0,3.0051466623942056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,4,128,1,float16,fp8,0,3.0069707234700522
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,24,128,1,float16,float16,0,1.719194730122884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,8,128,1,float16,float16,0,3.062666575113932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,24,128,1,float16,fp8,0,1.6523359616597493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,1,128,1,float16,float16,0,1.4021652539571126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,8,128,1,float16,fp8,0,3.263461430867513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,1,128,1,float16,fp8,0,1.6788372993469238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,2,128,1,float16,float16,0,1.4657227198282878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,4,128,1,float16,float16,0,1.5686987241109211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,2,128,1,float16,fp8,0,1.4919734001159668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,4,128,1,float16,fp8,0,1.550447940826416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,24,128,1,float16,float16,0,0.8882666428883871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,8,128,1,float16,float16,0,1.5290932655334473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,8,128,1,float16,fp8,0,1.605178674062093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,24,128,1,float16,fp8,0,0.8536213239034017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,1,128,1,float16,float16,0,0.7389866511027018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,1,128,1,float16,fp8,0,0.7522026697794596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,2,128,1,float16,float16,0,0.7438186804453532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,2,128,1,float16,fp8,0,0.778714656829834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,4,128,1,float16,float16,0,0.7822346687316895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,4,128,1,float16,fp8,0,0.8363093535105387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,8,128,1,float16,float16,0,0.8075359662373861
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,8,128,1,float16,fp8,0,0.8421546618143717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,1,128,1,float16,float16,0,6.635962804158528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,1,128,1,float16,fp8,0,6.460325241088867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,2,128,1,float16,float16,0,6.438581466674805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,2,128,1,float16,fp8,0,6.577194849650065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,4,128,1,float16,float16,0,7.679456075032552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,4,128,1,float16,fp8,0,7.119509379069011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,8,128,1,float16,float16,0,7.304426829020183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,1,128,1,float16,float16,0,3.3074827194213867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,24,128,1,float16,float16,0,4.063290596008301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,24,128,1,float16,fp8,0,3.7869014739990234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,1,128,1,float16,fp8,0,3.2646878560384116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,2,128,1,float16,float16,0,3.5560426712036133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,2,128,1,float16,fp8,0,3.2636534372965493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,4,128,1,float16,float16,0,3.6239360173543296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,8,128,1,float16,fp8,0,7.2454878489176435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,24,128,1,float16,float16,0,1.8722987174987793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,1,128,1,float16,float16,0,1.6425439516703289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,24,128,1,float16,fp8,0,2.0252906481424966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,4,128,1,float16,fp8,0,3.7277065912882485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,1,128,1,float16,fp8,0,1.7780267397562664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,8,128,1,float16,float16,0,3.5915679931640625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,8,128,1,float16,fp8,0,3.5836801528930664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,2,128,1,float16,fp8,0,1.706698735555013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,2,128,1,float16,float16,0,1.6494399706522624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,4,128,1,float16,float16,0,1.8185067176818848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,4,128,1,float16,fp8,0,1.8221653302510579
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,8,128,1,float16,float16,0,1.788032054901123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,24,128,1,float16,fp8,0,0.9924639860788981
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,24,128,1,float16,float16,0,0.9915359814961752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,8,128,1,float16,fp8,0,1.9106772740681965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,1,128,1,float16,float16,0,0.8223146597544352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,2,128,1,float16,fp8,0,0.8620853424072266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,1,128,1,float16,fp8,0,0.865290641784668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,2,128,1,float16,float16,0,0.8934346834818522
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,4,128,1,float16,fp8,0,0.9360373020172119
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,4,128,1,float16,float16,0,0.9053813616434733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,24,128,1,float16,float16,0,0.5132160186767578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,8,128,1,float16,float16,0,0.9342666467030843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,8,128,1,float16,fp8,0,0.975541353225708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,24,128,1,float16,fp8,0,0.5344213247299194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,2,128,1,float16,fp8,0,0.47092799345652264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,1,128,1,float16,float16,0,0.43432533740997314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,1,128,1,float16,fp8,0,0.4689120054244995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,2,128,1,float16,float16,0,0.4426399866739909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,4,128,1,float16,float16,0,0.47622398535410565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,4,128,1,float16,fp8,0,0.5113546848297119
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,8,128,1,float16,float16,0,0.4933493137359619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,8,128,1,float16,fp8,0,0.511685331662496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,1,128,1,float16,float16,0,6.427066802978516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,1,128,1,float16,fp8,0,6.192127863566081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,2,128,1,float16,float16,0,6.7392425537109375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,2,128,1,float16,fp8,0,6.237146377563477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,4,128,1,float16,float16,0,7.088815689086914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,4,128,1,float16,fp8,0,7.081663767496745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,8,128,1,float16,float16,0,7.326645533243815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,8,128,1,float16,fp8,0,6.978005091349284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,24,128,1,float16,float16,0,3.7024319966634116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,1,128,1,float16,float16,0,3.2608960469563804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,1,128,1,float16,fp8,0,3.0374132792154946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,24,128,1,float16,fp8,0,3.8104588190714517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,2,128,1,float16,float16,0,3.1475467681884766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,2,128,1,float16,fp8,0,3.114255905151367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,4,128,1,float16,float16,0,3.4538399378458657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,4,128,1,float16,fp8,0,3.5299253463745117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,24,128,1,float16,float16,0,1.8705813090006511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,1,128,1,float16,float16,0,1.5748799641927083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,24,128,1,float16,fp8,0,1.9221973419189453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,8,128,1,float16,float16,0,3.435471852620443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,1,128,1,float16,fp8,0,1.5426559448242188
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,8,128,1,float16,fp8,0,3.567978541056315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,2,128,1,float16,float16,0,1.6013706525166829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,2,128,1,float16,fp8,0,1.582090695699056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,4,128,1,float16,float16,0,1.7353173891703289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,24,128,1,float16,float16,0,0.9674239953358968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,4,128,1,float16,fp8,0,1.7768747011820476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,8,128,1,float16,float16,0,1.8715945879618328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,8,128,1,float16,fp8,0,1.8169013659159343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,24,128,1,float16,fp8,0,1.0258986949920654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,1,128,1,float16,float16,0,0.7675093015034994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,2,128,1,float16,float16,0,0.8171040217081705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,1,128,1,float16,fp8,0,0.8126239776611328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,2,128,1,float16,fp8,0,0.811568021774292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,4,128,1,float16,float16,0,0.9123040040334066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,4,128,1,float16,fp8,0,0.917626698811849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,8,128,1,float16,float16,0,0.9019733270009359
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,24,128,1,float16,float16,0,0.5048266649246216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,8,128,1,float16,fp8,0,0.9241546789805094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,24,128,1,float16,fp8,0,0.5328799883524576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,1,128,1,float16,float16,0,0.3933226664861043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,1,128,1,float16,fp8,0,0.43993600209554035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,2,128,1,float16,float16,0,0.414901336034139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,2,128,1,float16,fp8,0,0.44012268384297687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,4,128,1,float16,float16,0,0.4643893241882324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,4,128,1,float16,fp8,0,0.4822880029678345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,8,128,1,float16,float16,0,0.47547733783721924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,8,128,1,float16,fp8,0,0.4922506809234619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,1,128,1,float16,fp8,0,0.23594667514165243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,24,128,1,float16,float16,0,0.2698986728986104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,24,128,1,float16,fp8,0,0.2869759996732076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,1,128,1,float16,float16,0,0.2206826607386271
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,2,128,1,float16,float16,0,0.2312320073445638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,2,128,1,float16,fp8,0,0.2445546587308248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,4,128,1,float16,float16,0,0.2444960077603658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,4,128,1,float16,fp8,0,0.27395200729370117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,8,128,1,float16,float16,0,0.2553226749102275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,8,128,1,float16,fp8,0,0.2722346584002177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,1,128,1,float16,float16,0,3.560645421346029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,1,128,1,float16,fp8,0,3.6702025731404624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,2,128,1,float16,float16,0,3.667359987894694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,2,128,1,float16,fp8,0,3.8107999165852866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,4,128,1,float16,float16,0,4.231280008951823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,8,128,1,float16,float16,0,4.194538752237956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,8,128,1,float16,fp8,0,4.489967981974284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,1,128,1,float16,float16,0,1.7770026524861653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,24,128,1,float16,float16,0,2.4765332539876304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,1,128,1,float16,fp8,0,1.8696479797363281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,24,128,1,float16,fp8,0,2.4219199816385903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,2,128,1,float16,float16,0,1.9173760414123535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,2,128,1,float16,fp8,0,1.9736000696818035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,4,128,1,float16,float16,0,2.0985867182413735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,4,128,1,float16,fp8,0,4.438362757364909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,8,128,1,float16,float16,0,2.1344106992085776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,24,128,1,float16,float16,0,1.16921599706014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,24,128,1,float16,fp8,0,1.2734026908874512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,4,128,1,float16,fp8,0,2.314367930094401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,8,128,1,float16,fp8,0,2.2151999473571777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,1,128,1,float16,float16,0,0.9001226425170898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,1,128,1,float16,fp8,0,0.9543146292368571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,2,128,1,float16,fp8,0,1.000480016072591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,2,128,1,float16,float16,0,0.948853333791097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,4,128,1,float16,float16,0,1.0637226899464924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,4,128,1,float16,fp8,0,1.1541546980539958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,8,128,1,float16,float16,0,1.0839093526204426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,24,128,1,float16,float16,0,0.598688006401062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,8,128,1,float16,fp8,0,1.1362293561299641
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,24,128,1,float16,fp8,0,0.6493066549301147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,1,128,1,float16,float16,0,0.4665759801864624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,1,128,1,float16,fp8,0,0.4892959992090861
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,2,128,1,float16,float16,0,0.493066668510437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,2,128,1,float16,fp8,0,0.5180266698201498
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,4,128,1,float16,float16,0,0.5487146774927775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,8,128,1,float16,float16,0,0.547978679339091
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,4,128,1,float16,fp8,0,0.6000639994939169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,8,128,1,float16,fp8,0,0.6116319894790649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,24,128,1,float16,float16,0,0.32170132795969647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,24,128,1,float16,fp8,0,0.3436746597290039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,1,128,1,float16,float16,0,0.25085333983103436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,1,128,1,float16,fp8,0,0.2715733249982198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,2,128,1,float16,float16,0,0.26528533299763996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,2,128,1,float16,fp8,0,0.2749920090039571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,4,128,1,float16,float16,0,0.2955733338991801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,4,128,1,float16,fp8,0,0.32054932912190753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,8,128,1,float16,float16,0,0.3014933268229167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,8,128,1,float16,fp8,0,0.3224640091260274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,24,128,1,float16,float16,0,0.17993066708246866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,24,128,1,float16,fp8,0,0.19804799556732178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,1,128,1,float16,float16,0,0.14422399799029031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,1,128,1,float16,fp8,0,0.14962133765220642
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,2,128,1,float16,float16,0,0.1486186683177948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,2,128,1,float16,fp8,0,0.1564533313115438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,4,128,1,float16,float16,0,0.16591999928156534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,4,128,1,float16,fp8,0,0.1766186753908793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,8,128,1,float16,float16,0,0.16859734058380127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,8,128,1,float16,fp8,0,0.184879998366038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,1,128,1,float16,float16,0,3.484341303507487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,1,128,1,float16,fp8,0,3.681642532348633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,2,128,1,float16,float16,0,3.684373219807943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,2,128,1,float16,fp8,0,3.7942771911621094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,4,128,1,float16,float16,0,4.3187306722005205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,4,128,1,float16,fp8,0,4.6343733469645185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,8,128,1,float16,float16,0,4.359567960103353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,8,128,1,float16,fp8,0,4.566277186075847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,24,128,1,float16,float16,0,2.441317399342855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,1,128,1,float16,fp8,0,1.8736693064371746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,1,128,1,float16,float16,0,1.730021317799886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,2,128,1,float16,float16,0,1.8314132690429688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,24,128,1,float16,fp8,0,2.672885258992513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,2,128,1,float16,fp8,0,1.9477653503417969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,4,128,1,float16,float16,0,2.177882671356201
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,4,128,1,float16,fp8,0,2.3704479535420737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,8,128,1,float16,float16,0,2.1794986724853516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,1,128,1,float16,float16,0,0.8786559899648031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,24,128,1,float16,float16,0,1.2257440090179443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,1,128,1,float16,fp8,0,0.9368693033854166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,24,128,1,float16,fp8,0,1.3499360084533691
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,8,128,1,float16,fp8,0,2.3455732663472495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,2,128,1,float16,fp8,0,0.9863519668579102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,2,128,1,float16,float16,0,0.947551965713501
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,4,128,1,float16,float16,0,1.0918560028076172
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,8,128,1,float16,float16,0,1.1312692960103352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,24,128,1,float16,float16,0,0.6275733311971029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,4,128,1,float16,fp8,0,1.1960533459981282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,8,128,1,float16,fp8,0,1.218015988667806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,1,128,1,float16,float16,0,0.4532053470611572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,24,128,1,float16,fp8,0,0.6891626516977946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,1,128,1,float16,fp8,0,0.4907253185908
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,2,128,1,float16,float16,0,0.494917352994283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,2,128,1,float16,fp8,0,0.511738657951355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,4,128,1,float16,float16,0,0.56113068262736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,4,128,1,float16,fp8,0,0.6106186707814535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,8,128,1,float16,float16,0,0.5721493164698283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,8,128,1,float16,fp8,0,0.6354399919509888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,24,128,1,float16,float16,0,0.3309706648190816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,2,128,1,float16,float16,0,0.2571733395258586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,24,128,1,float16,fp8,0,0.3606933355331421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,1,128,1,float16,float16,0,0.24169600009918213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,1,128,1,float16,fp8,0,0.260917325814565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,2,128,1,float16,fp8,0,0.2760213414827983
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,4,128,1,float16,float16,0,0.29662932952245075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,4,128,1,float16,fp8,0,0.32811200618743896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,8,128,1,float16,float16,0,0.3056640028953552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,8,128,1,float16,fp8,0,0.3312693238258362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,24,128,1,float16,float16,0,0.1839146614074707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,24,128,1,float16,fp8,0,0.19912532965342203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,1,128,1,float16,float16,0,0.13397866487503052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,1,128,1,float16,fp8,0,0.14615999658902487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,2,128,1,float16,float16,0,0.14313066999117532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,2,128,1,float16,fp8,0,0.1541920006275177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,4,128,1,float16,float16,0,0.1646719972292582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,4,128,1,float16,fp8,0,0.1779359976450602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,8,128,1,float16,float16,0,0.17083734273910522
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,8,128,1,float16,fp8,0,0.18571732441584268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,24,128,1,float16,float16,0,0.10417067011197408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,24,128,1,float16,fp8,0,0.11855467160542806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,1,128,1,float16,float16,0,0.07241066793600719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,1,128,1,float16,fp8,0,0.08042133351167043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,2,128,1,float16,float16,0,0.08010666569073994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,2,128,1,float16,fp8,0,0.08574400345484416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,4,128,1,float16,float16,0,0.08992000420888265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,4,128,1,float16,fp8,0,0.09696533282597859
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,8,128,1,float16,float16,0,0.09565866986910503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,8,128,1,float16,fp8,0,0.10333866874376933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,1,128,1,float16,float16,0,2.0899945894877114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,1,128,1,float16,fp8,0,2.2962560653686523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,2,128,1,float16,float16,0,2.3044959704081216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,2,128,1,float16,fp8,0,2.413344065348307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,4,128,1,float16,float16,0,2.767423947652181
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,4,128,1,float16,fp8,0,3.0479307174682617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,8,128,1,float16,float16,0,2.7929118474324546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,24,128,1,float16,float16,0,1.5816319783528645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,8,128,1,float16,fp8,0,3.032128016153971
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,1,128,1,float16,float16,0,1.0743146737416585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,24,128,1,float16,fp8,0,1.7592479387919109
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,1,128,1,float16,fp8,0,1.1598560015360515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,2,128,1,float16,float16,0,1.1819519996643066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,2,128,1,float16,fp8,0,1.2268106937408447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,4,128,1,float16,float16,0,1.3804052670796711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,24,128,1,float16,float16,0,0.806821346282959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,8,128,1,float16,float16,0,1.404197374979655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,4,128,1,float16,fp8,0,1.5434400240580242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,24,128,1,float16,fp8,0,0.892794688542684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,8,128,1,float16,fp8,0,1.5720373789469402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,1,128,1,float16,float16,0,0.5527786811192831
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,1,128,1,float16,fp8,0,0.5943146546681722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,2,128,1,float16,float16,0,0.601861317952474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,2,128,1,float16,fp8,0,0.6256693204243978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,4,128,1,float16,float16,0,0.69923202196757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,4,128,1,float16,fp8,0,0.7984853585561117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,24,128,1,float16,float16,0,0.42104001839955646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,8,128,1,float16,fp8,0,0.8087626298268636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,1,128,1,float16,fp8,0,0.31351999441782635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,8,128,1,float16,float16,0,0.7126133441925049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,24,128,1,float16,fp8,0,0.4607573350270589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,1,128,1,float16,float16,0,0.2881866693496704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,4,128,1,float16,fp8,0,0.4143679936726888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,2,128,1,float16,float16,0,0.31243733565012616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,2,128,1,float16,fp8,0,0.33505598704020184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,4,128,1,float16,float16,0,0.3733439842859904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,8,128,1,float16,float16,0,0.37309332688649494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,8,128,1,float16,fp8,0,0.41725865999857586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,1,128,1,float16,fp8,0,0.17617066701253256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,24,128,1,float16,float16,0,0.22266666094462076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,24,128,1,float16,fp8,0,0.24991466601689658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,1,128,1,float16,float16,0,0.15930666526158652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,2,128,1,float16,float16,0,0.17492800951004028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,2,128,1,float16,fp8,0,0.18197866280873617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,4,128,1,float16,float16,0,0.19985600312550864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,4,128,1,float16,fp8,0,0.2214933236440023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,8,128,1,float16,float16,0,0.2031466762224833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,24,128,1,float16,float16,0,0.12567466497421265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,8,128,1,float16,fp8,0,0.23317867517471313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,24,128,1,float16,fp8,0,0.14109333356221518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,1,128,1,float16,float16,0,0.08749333024024963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,1,128,1,float16,fp8,0,0.09662399689356486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,2,128,1,float16,float16,0,0.09713066617647807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,2,128,1,float16,fp8,0,0.10249599814414978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,4,128,1,float16,float16,0,0.11009599765141805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,4,128,1,float16,fp8,0,0.12222933769226074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,8,128,1,float16,float16,0,0.11507733662923177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,8,128,1,float16,fp8,0,0.12763200203577676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,24,128,1,float16,float16,0,0.07454933226108551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,24,128,1,float16,fp8,0,0.0842133363087972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,1,128,1,float16,float16,0,0.053114667534828186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,1,128,1,float16,fp8,0,0.06065066655476888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,2,128,1,float16,float16,0,0.054101333022117615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,2,128,1,float16,fp8,0,0.062309334675470986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,4,128,1,float16,float16,0,0.060549333691596985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,4,128,1,float16,fp8,0,0.06797866523265839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,8,128,1,float16,float16,0,0.06398400167624156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,8,128,1,float16,fp8,0,0.07069866855939229
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,1,128,1,float16,float16,0,2.2091520627339682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,1,128,1,float16,fp8,0,2.470165411631266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,2,128,1,float16,float16,0,2.484997272491455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,2,128,1,float16,fp8,0,2.5925866762797036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,8,128,1,float16,float16,0,3.075519879659017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,4,128,1,float16,fp8,0,3.452885309855143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,4,128,1,float16,float16,0,3.0459518432617188
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,24,128,1,float16,float16,0,1.796138604482015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,1,128,1,float16,float16,0,1.114192008972168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,1,128,1,float16,fp8,0,1.2529760201772053
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,24,128,1,float16,fp8,0,2.1166186332702637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,8,128,1,float16,fp8,0,3.438058535257975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,2,128,1,float16,float16,0,1.2401706377665203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,2,128,1,float16,fp8,0,1.3227039972941081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,24,128,1,float16,float16,0,0.9207893212636312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,8,128,1,float16,float16,0,1.544149398803711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,4,128,1,float16,float16,0,1.5502293904622395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,24,128,1,float16,fp8,0,1.0426080226898193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,1,128,1,float16,float16,0,0.5745813449223837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,4,128,1,float16,fp8,0,1.7423946062723796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,8,128,1,float16,fp8,0,1.7661760648091633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,1,128,1,float16,fp8,0,0.6362400054931641
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,2,128,1,float16,float16,0,0.6496906677881876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,2,128,1,float16,fp8,0,0.6719679832458496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,4,128,1,float16,float16,0,0.7736372947692871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,8,128,1,float16,float16,0,0.7892639636993408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,4,128,1,float16,fp8,0,0.8828746477762858
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,24,128,1,float16,float16,0,0.467306653658549
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,24,128,1,float16,fp8,0,0.5362933476765951
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,1,128,1,float16,float16,0,0.30379199981689453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,1,128,1,float16,fp8,0,0.3306986689567566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,2,128,1,float16,fp8,0,0.3504800001780192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,2,128,1,float16,float16,0,0.3359146515528361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,8,128,1,float16,float16,0,0.41606398423512775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,4,128,1,float16,float16,0,0.4056853453318278
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,8,128,1,float16,fp8,0,0.9005066553751627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,24,128,1,float16,fp8,0,0.2804639935493469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,4,128,1,float16,fp8,0,0.4590666691462199
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,24,128,1,float16,float16,0,0.24888533353805542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,8,128,1,float16,fp8,0,0.48077865441640216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,1,128,1,float16,float16,0,0.15899200240770975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,1,128,1,float16,fp8,0,0.18175466855367026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,2,128,1,float16,float16,0,0.18068265914916992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,2,128,1,float16,fp8,0,0.19183999300003052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,4,128,1,float16,float16,0,0.21428799629211426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,4,128,1,float16,fp8,0,0.24326932430267334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,8,128,1,float16,float16,0,0.2214933236440023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,8,128,1,float16,fp8,0,0.2559573252995809
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,24,128,1,float16,float16,0,0.13312533497810364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,24,128,1,float16,fp8,0,0.1553973356882731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,1,128,1,float16,float16,0,0.09093333284060161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,1,128,1,float16,fp8,0,0.10299733281135559
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,2,128,1,float16,float16,0,0.10014399886131287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,2,128,1,float16,fp8,0,0.10993066430091858
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,4,128,1,float16,float16,0,0.11779733498891194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,4,128,1,float16,fp8,0,0.13618133465449014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,1,128,1,float16,float16,0,0.05129600067933401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,8,128,1,float16,fp8,0,0.14135467012723288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,8,128,1,float16,float16,0,0.12083199620246887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,24,128,1,float16,float16,0,0.07755200068155925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,24,128,1,float16,fp8,0,0.09165866176287334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,1,128,1,float16,fp8,0,0.05728533367315928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,8,128,1,float16,float16,0,0.06899733344713847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,8,128,1,float16,fp8,0,0.07889066636562347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,2,128,1,float16,float16,0,0.054527997970581055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,2,128,1,float16,fp8,0,0.06271466612815857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,4,128,1,float16,float16,0,0.06347199777762096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,4,128,1,float16,fp8,0,0.07012266914049785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,24,128,1,float16,float16,0,0.047882666190465294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,24,128,1,float16,fp8,0,0.05176533261934916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,1,128,1,float16,float16,0,0.03422933320204417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,1,128,1,float16,fp8,0,0.03857066730658213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,2,128,1,float16,float16,0,0.03568000098069509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,2,128,1,float16,fp8,0,0.040421334405740104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,4,128,1,float16,float16,0,0.03979733337958654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,4,128,1,float16,fp8,0,0.04504533112049103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,8,128,1,float16,float16,0,0.03941866755485535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,8,128,1,float16,fp8,0,0.046351999044418335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,1,128,1,float16,float16,0,1.645898660024007
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,1,128,1,float16,fp8,0,1.8904959360758464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,2,128,1,float16,float16,0,1.907973289489746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,2,128,1,float16,fp8,0,2.0362399419148765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,4,128,1,float16,float16,0,2.4800373713175454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,24,128,1,float16,float16,0,1.5239040056864421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,4,128,1,float16,fp8,0,2.88863468170166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,1,128,1,float16,float16,0,0.8312959671020508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,24,128,1,float16,fp8,0,1.8361652692159016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,8,128,1,float16,float16,0,2.527088006337484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,8,128,1,float16,fp8,0,2.929903984069824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,2,128,1,float16,float16,0,0.9657493432362875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,1,128,1,float16,fp8,0,0.9603306452433268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,2,128,1,float16,fp8,0,1.0329386393229167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,4,128,1,float16,float16,0,1.2472426891326904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,8,128,1,float16,float16,0,1.2544159889221191
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,24,128,1,float16,float16,0,0.7779413064320883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,4,128,1,float16,fp8,0,1.4670987129211426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,1,128,1,float16,float16,0,0.4288373390833537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,24,128,1,float16,fp8,0,0.9329173564910889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,8,128,1,float16,fp8,0,1.4797226587931316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,1,128,1,float16,fp8,0,0.48764268557230633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,4,128,1,float16,float16,0,0.6366986831029257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,2,128,1,float16,fp8,0,0.5317173401514689
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,24,128,1,float16,fp8,0,0.476911981900533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,2,128,1,float16,float16,0,0.49668800830841064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,8,128,1,float16,float16,0,0.6415786743164062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,1,128,1,float16,float16,0,0.22472532590230307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,1,128,1,float16,fp8,0,0.25723199049631756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,24,128,1,float16,float16,0,0.3956960042317708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,8,128,1,float16,fp8,0,0.7917706966400146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,2,128,1,float16,fp8,0,0.2786293427149455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,2,128,1,float16,float16,0,0.2674186627070109
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,4,128,1,float16,float16,0,0.32712000608444214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,4,128,1,float16,fp8,0,0.7528853416442871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,4,128,1,float16,fp8,0,0.38653333981831867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,8,128,1,float16,fp8,0,0.40200531482696533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,1,128,1,float16,float16,0,0.12343999743461609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,24,128,1,float16,fp8,0,0.2508106629053752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,8,128,1,float16,float16,0,0.3386133511861165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,1,128,1,float16,fp8,0,0.14244799812634787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,24,128,1,float16,float16,0,0.20758932828903198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,2,128,1,float16,float16,0,0.14378666877746582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,2,128,1,float16,fp8,0,0.15254400173823038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,4,128,1,float16,float16,0,0.17356266578038534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,8,128,1,float16,float16,0,0.17999466260274252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,4,128,1,float16,fp8,0,0.20659732818603516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,8,128,1,float16,fp8,0,0.216538667678833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,24,128,1,float16,float16,0,0.11319466431935628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,24,128,1,float16,fp8,0,0.13987732927004495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,1,128,1,float16,float16,0,0.07251733541488647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,1,128,1,float16,fp8,0,0.0832479993502299
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,2,128,1,float16,float16,0,0.07967466612656911
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,2,128,1,float16,fp8,0,0.08810133735338847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,4,128,1,float16,float16,0,0.09624532858530681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,4,128,1,float16,fp8,0,0.11343999703725179
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,8,128,1,float16,float16,0,0.09866133332252502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,24,128,1,float16,float16,0,0.06709333260854085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,24,128,1,float16,fp8,0,0.08000533282756805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,8,128,1,float16,fp8,0,0.12172266840934753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,1,128,1,float16,float16,0,0.04028266668319702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,1,128,1,float16,fp8,0,0.04578666885693868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,2,128,1,float16,float16,0,0.04261866708596548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,8,128,1,float16,fp8,0,0.06404800216356914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,2,128,1,float16,fp8,0,0.0495413343111674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,4,128,1,float16,float16,0,0.05049066742261251
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,4,128,1,float16,fp8,0,0.057376002271970115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,8,128,1,float16,float16,0,0.0558240016301473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,24,128,1,float16,float16,0,0.03913066784540812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,24,128,1,float16,fp8,0,0.043477331598599754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,1,128,1,float16,float16,0,0.027242665489514668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,4,128,1,float16,fp8,0,0.03859733293453852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,1,128,1,float16,fp8,0,0.030799999833106995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,2,128,1,float16,float16,0,0.028309332827727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,2,128,1,float16,fp8,0,0.0337119996547699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,4,128,1,float16,float16,0,0.031119999786218006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,8,128,1,float16,float16,0,0.031013332307338715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,8,128,1,float16,fp8,0,0.0383840004603068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,2,128,1,float16,float16,0,0.020096000283956528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,24,128,1,float16,float16,0,0.022800001005331676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,24,128,1,float16,fp8,0,0.02808533360560735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,1,128,1,float16,float16,0,0.019637333850065868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,1,128,1,float16,fp8,0,0.022869333624839783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,2,128,1,float16,fp8,0,0.023621333142121632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,4,128,1,float16,float16,0,0.021482666333516438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,4,128,1,float16,fp8,0,0.026330667237440746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,8,128,1,float16,float16,0,0.02142400046189626
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,8,128,1,float16,fp8,0,0.027061333258946735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,1,128,1,float16,float16,0,0.7023466428120931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,1,128,1,float16,fp8,0,0.8175466855367025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,2,128,1,float16,float16,0,0.8372853597005209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,2,128,1,float16,fp8,0,0.8991359869639078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,4,128,1,float16,float16,0,1.117637316385905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,4,128,1,float16,fp8,0,1.3274986743927002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,24,128,1,float16,float16,0,0.7097439765930176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,8,128,1,float16,float16,0,1.1246240139007568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,1,128,1,float16,float16,0,0.3627893527348836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,24,128,1,float16,fp8,0,0.815877358118693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,8,128,1,float16,fp8,0,1.3408959706624348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,1,128,1,float16,fp8,0,0.4195093313852946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,2,128,1,float16,float16,0,0.42957866191864014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,2,128,1,float16,fp8,0,0.4654239813486735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,4,128,1,float16,fp8,0,0.6739947001139323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,4,128,1,float16,float16,0,0.5700106620788574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,24,128,1,float16,float16,0,0.3646080096562703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,8,128,1,float16,float16,0,0.5908639828364054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,24,128,1,float16,fp8,0,0.42049598693847656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,1,128,1,float16,float16,0,0.19140267372131348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,8,128,1,float16,fp8,0,0.713269313176473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,1,128,1,float16,fp8,0,0.22025066614151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,2,128,1,float16,float16,0,0.2265333334604899
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,2,128,1,float16,fp8,0,0.24308266242345175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,4,128,1,float16,float16,0,0.29337600866953534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,4,128,1,float16,fp8,0,0.35049064954121906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,24,128,1,float16,float16,0,0.19187732537587485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,24,128,1,float16,fp8,0,0.22266666094462076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,1,128,1,float16,float16,0,0.1051626702149709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,8,128,1,float16,fp8,0,0.3667253255844116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,1,128,1,float16,fp8,0,0.12231466174125671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,8,128,1,float16,float16,0,0.3030826648076375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,2,128,1,float16,float16,0,0.12126400073369344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,4,128,1,float16,float16,0,0.15616533160209656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,2,128,1,float16,fp8,0,0.13500266273816428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,4,128,1,float16,fp8,0,0.18665067354838052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,8,128,1,float16,float16,0,0.1612266699473063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,8,128,1,float16,fp8,0,0.19778666893641153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,24,128,1,float16,float16,0,0.10601066549619038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,24,128,1,float16,fp8,0,0.12359999616940816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,4,128,1,float16,float16,0,0.08617599805196126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,1,128,1,float16,float16,0,0.06140799820423126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,1,128,1,float16,fp8,0,0.07217066486676534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,2,128,1,float16,float16,0,0.06992533306280772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,2,128,1,float16,fp8,0,0.07885333398977916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,4,128,1,float16,fp8,0,0.10416533549626668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,8,128,1,float16,float16,0,0.09070400396982829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,24,128,1,float16,float16,0,0.061050668358802795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,8,128,1,float16,fp8,0,0.1088693340619405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,24,128,1,float16,fp8,0,0.0712960014740626
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,1,128,1,float16,float16,0,0.03257066756486893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,1,128,1,float16,fp8,0,0.0383093332250913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,4,128,1,float16,fp8,0,0.05188799897829691
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,2,128,1,float16,float16,0,0.035360001027584076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,2,128,1,float16,fp8,0,0.0418453315893809
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,4,128,1,float16,float16,0,0.045381332437197365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,8,128,1,float16,float16,0,0.04951466619968414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,8,128,1,float16,fp8,0,0.05705066521962484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,24,128,1,float16,float16,0,0.03392533212900162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,24,128,1,float16,fp8,0,0.03952533255020777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,1,128,1,float16,float16,0,0.022842665513356526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,1,128,1,float16,fp8,0,0.026650667190551758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,2,128,1,float16,float16,0,0.02370133250951767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,2,128,1,float16,fp8,0,0.028394666810830433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,4,128,1,float16,float16,0,0.026288000245889027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,24,128,1,float16,fp8,0,0.025055999557177227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,4,128,1,float16,fp8,0,0.03234666585922241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,8,128,1,float16,float16,0,0.027082666754722595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,8,128,1,float16,fp8,0,0.0329120010137558
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,24,128,1,float16,float16,0,0.019744000087181728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,1,128,1,float16,float16,0,0.0169813334941864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,4,128,1,float16,fp8,0,0.022778667509555817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,1,128,1,float16,fp8,0,0.019914666811625164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,2,128,1,float16,float16,0,0.017370666066805523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,2,128,1,float16,fp8,0,0.02014933278163274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,24,128,1,float16,fp8,0,0.022143999735514324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,4,128,1,float16,float16,0,0.018618666877349217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,8,128,1,float16,float16,0,0.01874133323629697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,8,128,1,float16,fp8,0,0.02311466634273529
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,24,128,1,float16,float16,0,0.017711999515692394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,1,128,1,float16,float16,0,0.01714666684468587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,1,128,1,float16,fp8,0,0.019296000401178997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,2,128,1,float16,float16,0,0.01718933383623759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,8,128,1,float16,fp8,0,0.021503999829292297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,2,128,1,float16,fp8,0,0.01970133309563001
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,4,128,1,float16,float16,0,0.016778666526079178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,4,128,1,float16,fp8,0,0.021525333325068157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,8,128,1,float16,float16,0,0.017045332739750545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,1,128,1,float16,float16,0,0.3470453421274821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,1,128,1,float16,fp8,0,0.373418649037679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,2,128,1,float16,float16,0,0.4017546574274699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,4,128,1,float16,float16,0,0.5517386595408121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,4,128,1,float16,fp8,0,0.6299786567687988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,2,128,1,float16,fp8,0,0.41948266824086505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,24,128,1,float16,float16,0,0.35028799374898273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,24,128,1,float16,fp8,0,0.3851360082626343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,8,128,1,float16,float16,0,0.570522665977478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,1,128,1,float16,float16,0,0.18202133973439535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,8,128,1,float16,fp8,0,0.6730079650878906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,1,128,1,float16,fp8,0,0.20009599129358926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,2,128,1,float16,float16,0,0.21840532620747885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,2,128,1,float16,fp8,0,0.2226933240890503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,8,128,1,float16,fp8,0,0.34062933921813965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,24,128,1,float16,fp8,0,0.20459733406702676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,1,128,1,float16,float16,0,0.09955199559529622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,8,128,1,float16,float16,0,0.29471999406814575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,24,128,1,float16,float16,0,0.18522665898005167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,4,128,1,float16,float16,0,0.2866400082906087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,4,128,1,float16,fp8,0,0.3263946572939555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,1,128,1,float16,fp8,0,0.11434132854143779
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,2,128,1,float16,float16,0,0.11762666702270508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,2,128,1,float16,fp8,0,0.12613333264986673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,8,128,1,float16,float16,0,0.15561599532763162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,4,128,1,float16,float16,0,0.15080533425013223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,24,128,1,float16,float16,0,0.10260799527168274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,24,128,1,float16,fp8,0,0.11101333300272624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,8,128,1,float16,fp8,0,0.18125333388646445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,1,128,1,float16,float16,0,0.05858666698137919
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,4,128,1,float16,fp8,0,0.17593600352605185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,1,128,1,float16,fp8,0,0.06586666901906331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,2,128,1,float16,float16,0,0.06817600131034851
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,8,128,1,float16,float16,0,0.08693333466847737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,4,128,1,float16,float16,0,0.08338666955629985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,4,128,1,float16,fp8,0,0.0962559978167216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,2,128,1,float16,fp8,0,0.0728053351243337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,8,128,1,float16,fp8,0,0.10039466619491577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,24,128,1,float16,float16,0,0.05804799993832906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,24,128,1,float16,fp8,0,0.06273599962393443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,1,128,1,float16,float16,0,0.02906133234500885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,1,128,1,float16,fp8,0,0.033413333197434746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,2,128,1,float16,float16,0,0.03173333406448364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,2,128,1,float16,fp8,0,0.03626133253177007
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,4,128,1,float16,float16,0,0.039077334105968475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,4,128,1,float16,fp8,0,0.04365866879622141
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,8,128,1,float16,float16,0,0.048469334840774536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,1,128,1,float16,float16,0,0.021594665944576263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,8,128,1,float16,fp8,0,0.04634666442871094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,24,128,1,float16,float16,0,0.03310933212439219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,24,128,1,float16,fp8,0,0.032986665765444435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,1,128,1,float16,fp8,0,0.024618667860825855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,2,128,1,float16,float16,0,0.022154666483402252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,4,128,1,float16,float16,0,0.025311999022960663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,4,128,1,float16,fp8,0,0.029098667204380035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,2,128,1,float16,fp8,0,0.025722667574882507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,8,128,1,float16,float16,0,0.02499199906984965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,8,128,1,float16,fp8,0,0.029322666426499683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,24,128,1,float16,float16,0,0.017957333475351334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,24,128,1,float16,fp8,0,0.02063999945918719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,1,128,1,float16,float16,0,0.015637333194414776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,1,128,1,float16,fp8,0,0.01821333294113477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,2,128,1,float16,float16,0,0.016202667107184727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,2,128,1,float16,fp8,0,0.018250666558742523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,4,128,1,float16,float16,0,0.017050666113694508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,4,128,1,float16,fp8,0,0.019914666811625164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,8,128,1,float16,float16,0,0.017279999951521557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,8,128,1,float16,fp8,0,0.020037333170572918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,24,128,1,float16,float16,0,0.01632000009218852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,24,128,1,float16,fp8,0,0.01844266677896182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,1,128,1,float16,float16,0,0.01516266663869222
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,1,128,1,float16,fp8,0,0.017808000246683758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,2,128,1,float16,float16,0,0.01543466622630755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,2,128,1,float16,fp8,0,0.017514667163292568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,4,128,1,float16,float16,0,0.015685333559910457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,4,128,1,float16,fp8,0,0.01788266624013583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,8,128,1,float16,float16,0,0.015354666858911514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,8,128,1,float16,fp8,0,0.01823466643691063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,2,128,1,float16,fp8,0,0.017562666287024815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,24,128,1,float16,float16,0,0.015599999576807022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,24,128,1,float16,fp8,0,0.017738666385412216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,1,128,1,float16,float16,0,0.014917333920796713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,2,128,1,float16,float16,0,0.01498666654030482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,1,128,1,float16,fp8,0,0.017557332913080852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,4,128,1,float16,float16,0,0.01471466695268949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,4,128,1,float16,fp8,0,0.01773333301146825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,8,128,1,float16,float16,0,0.01532799998919169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,8,128,1,float16,fp8,0,0.01740266631046931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,1,128,1,float16,float16,0,0.18266133467356363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,1,128,1,float16,fp8,0,0.19975467522939047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,2,128,1,float16,float16,0,0.2135466734568278
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,4,128,1,float16,float16,0,0.2819040020306905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,4,128,1,float16,fp8,0,0.3292906681696574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,24,128,1,float16,float16,0,0.20151466131210327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,2,128,1,float16,fp8,0,0.22324800491333008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,1,128,1,float16,float16,0,0.09855467081069946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,24,128,1,float16,fp8,0,0.20033599932988486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,8,128,1,float16,float16,0,0.3269279996554057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,8,128,1,float16,fp8,0,0.3735040028889974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,1,128,1,float16,fp8,0,0.11398933331171672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,2,128,1,float16,float16,0,0.11808000008265178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,4,128,1,float16,fp8,0,0.1745120088259379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,4,128,1,float16,float16,0,0.15026666720708212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,2,128,1,float16,fp8,0,0.1273973286151886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,8,128,1,float16,float16,0,0.1735573410987854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,24,128,1,float16,fp8,0,0.10943466424942017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,24,128,1,float16,float16,0,0.10955199599266052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,8,128,1,float16,fp8,0,0.2015893260637919
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,1,128,1,float16,float16,0,0.05932266513506571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,1,128,1,float16,fp8,0,0.06541866560777028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,2,128,1,float16,float16,0,0.06718400120735168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,2,128,1,float16,fp8,0,0.07377066711584727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,4,128,1,float16,float16,0,0.08431466420491536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,4,128,1,float16,fp8,0,0.09621333082516988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,8,128,1,float16,float16,0,0.09657599528630574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,24,128,1,float16,float16,0,0.062458669145902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,8,128,1,float16,fp8,0,0.10966933767000835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,24,128,1,float16,fp8,0,0.059445331494013466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,1,128,1,float16,float16,0,0.028592000404993694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,1,128,1,float16,fp8,0,0.03311999887228012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,2,128,1,float16,float16,0,0.03181866556406021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,2,128,1,float16,fp8,0,0.03596800069014231
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,4,128,1,float16,float16,0,0.03899200012286504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,4,128,1,float16,fp8,0,0.04354133208592733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,8,128,1,float16,float16,0,0.052101333936055504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,8,128,1,float16,fp8,0,0.053413331508636475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,24,128,1,float16,float16,0,0.032101333141326904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,24,128,1,float16,fp8,0,0.02802666773398717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,1,128,1,float16,float16,0,0.021349333226680756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,1,128,1,float16,fp8,0,0.024405332903067272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,2,128,1,float16,float16,0,0.0222080002228419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,2,128,1,float16,fp8,0,0.025727999707063038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,4,128,1,float16,float16,0,0.024725332856178284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,4,128,1,float16,fp8,0,0.02903999884923299
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,8,128,1,float16,float16,0,0.024800000091393787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,8,128,1,float16,fp8,0,0.029098667204380035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,24,128,1,float16,float16,0,0.019343999524911244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,24,128,1,float16,fp8,0,0.019968000551064808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,1,128,1,float16,float16,0,0.01545599972208341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,1,128,1,float16,fp8,0,0.0183999997874101
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,4,128,1,float16,fp8,0,0.020037333170572918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,2,128,1,float16,float16,0,0.01581866666674614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,2,128,1,float16,fp8,0,0.018320000420014065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,4,128,1,float16,float16,0,0.017152000218629837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,8,128,1,float16,float16,0,0.017136000096797943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,8,128,1,float16,fp8,0,0.019813333948453266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,24,128,1,float16,float16,0,0.014416000495354334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,24,128,1,float16,fp8,0,0.014405333747466406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,2,128,1,float16,fp8,0,0.017610666652520496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,1,128,1,float16,float16,0,0.014874666929244995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,4,128,1,float16,fp8,0,0.017903999735911686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,1,128,1,float16,fp8,0,0.0179626668492953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,2,128,1,float16,float16,0,0.015861333658297855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,4,128,1,float16,float16,0,0.015696000307798386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,8,128,1,float16,float16,0,0.015466666469971338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,8,128,1,float16,fp8,0,0.01811733345190684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,24,128,1,float16,float16,0,0.013861333330472311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,24,128,1,float16,fp8,0,0.013471999516089758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,1,128,1,float16,float16,0,0.015013333410024643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,1,128,1,float16,fp8,0,0.017466666797796886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,2,128,1,float16,float16,0,0.015370666980743408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,2,128,1,float16,fp8,0,0.017509333789348602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,4,128,1,float16,float16,0,0.015157333264748255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,4,128,1,float16,fp8,0,0.017616000026464462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,1,128,1,float16,fp8,0,0.016250666230916977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,8,128,1,float16,float16,0,0.01515199989080429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,8,128,1,float16,fp8,0,0.017642666896184284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,24,128,1,float16,float16,0,0.013232000172138214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,24,128,1,float16,fp8,0,0.01392000044385592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,1,128,1,float16,float16,0,0.014346666634082794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,2,128,1,float16,float16,0,0.014282666146755219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,2,128,1,float16,fp8,0,0.016645333419243496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,4,128,1,float16,float16,0,0.014346666634082794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,4,128,1,float16,fp8,0,0.016565332810084026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,8,128,1,float16,float16,0,0.014303999642531076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,8,128,1,float16,fp8,0,0.0174346665541331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,1,128,1,float16,float16,0,0.09756799538930257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,1,128,1,float16,fp8,0,0.1153546671072642
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,2,128,1,float16,float16,0,0.11727466185887654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,2,128,1,float16,fp8,0,0.12575999895731607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,4,128,1,float16,float16,0,0.1665386656920115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,4,128,1,float16,fp8,0,0.19555199146270752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,8,128,1,float16,float16,0,0.17408533891042074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,24,128,1,float16,float16,0,0.14539200067520142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,8,128,1,float16,fp8,0,0.20059200127919516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,24,128,1,float16,fp8,0,0.14205333590507507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,1,128,1,float16,float16,0,0.05890133480230967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,1,128,1,float16,fp8,0,0.06498666604359944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,2,128,1,float16,float16,0,0.06702933212121327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,2,128,1,float16,fp8,0,0.07339733342329662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,4,128,1,float16,float16,0,0.09238400061925252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,24,128,1,float16,float16,0,0.07937600215276082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,8,128,1,float16,float16,0,0.09566400448481242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,24,128,1,float16,fp8,0,0.07831466694672902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,4,128,1,float16,fp8,0,0.1102239986260732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,2,128,1,float16,float16,0,0.03137599925200144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,8,128,1,float16,fp8,0,0.1095199982325236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,2,128,1,float16,fp8,0,0.03558400024970373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,1,128,1,float16,float16,0,0.028277332584063213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,1,128,1,float16,fp8,0,0.033226666351159416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,4,128,1,float16,float16,0,0.04318933188915253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,4,128,1,float16,fp8,0,0.04909333089987437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,8,128,1,float16,float16,0,0.05156800150871277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,8,128,1,float16,fp8,0,0.05243200063705444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,24,128,1,float16,float16,0,0.04162666698296865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,24,128,1,float16,fp8,0,0.03834133346875509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,1,128,1,float16,float16,0,0.02089600016673406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,1,128,1,float16,fp8,0,0.024485332270463307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,2,128,1,float16,float16,0,0.022101332743962605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,2,128,1,float16,fp8,0,0.025573333104451496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,4,128,1,float16,float16,0,0.024725332856178284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,4,128,1,float16,fp8,0,0.028666667640209198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,8,128,1,float16,float16,0,0.024714666108290356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,1,128,1,float16,fp8,0,0.018298666924238205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,8,128,1,float16,fp8,0,0.028970666229724884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,24,128,1,float16,float16,0,0.02254933367172877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,24,128,1,float16,fp8,0,0.02317333221435547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,1,128,1,float16,float16,0,0.015290666371583939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,4,128,1,float16,fp8,0,0.019610666980346043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,2,128,1,float16,float16,0,0.015647999942302704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,4,128,1,float16,float16,0,0.016949333250522614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,2,128,1,float16,fp8,0,0.018250666558742523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,8,128,1,float16,float16,0,0.016757333030303318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,8,128,1,float16,fp8,0,0.019893333315849304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,24,128,1,float16,float16,0,0.01710933322707812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,24,128,1,float16,fp8,0,0.017386666188637417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,2,128,1,float16,fp8,0,0.017573333034912746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,1,128,1,float16,float16,0,0.01523200049996376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,1,128,1,float16,fp8,0,0.017525333911180496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,2,128,1,float16,float16,0,0.015146666516860327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,4,128,1,float16,float16,0,0.015370666980743408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,4,128,1,float16,fp8,0,0.017605333278576534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,8,128,1,float16,float16,0,0.01543466622630755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,8,128,1,float16,fp8,0,0.017616000026464462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,24,128,1,float16,float16,0,0.013461332768201828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,24,128,1,float16,fp8,0,0.014058666924635569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,1,128,1,float16,float16,0,0.015114666273196539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,1,128,1,float16,fp8,0,0.017637333522240322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,2,128,1,float16,float16,0,0.014752000570297241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,2,128,1,float16,fp8,0,0.017397332936525345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,4,128,1,float16,float16,0,0.01488000030318896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,4,128,1,float16,fp8,0,0.017621333400408428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,8,128,1,float16,float16,0,0.014805333067973455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,8,128,1,float16,fp8,0,0.01717866708834966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,24,128,1,float16,float16,0,0.01313599944114685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,24,128,1,float16,fp8,0,0.013594667116800943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,1,128,1,float16,float16,0,0.01423466702302297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,1,128,1,float16,fp8,0,0.01624533285697301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,2,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,2,128,1,float16,fp8,0,0.01642666632930438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,4,128,1,float16,float16,0,0.014442666123310724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,4,128,1,float16,fp8,0,0.0162773331006368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,8,128,1,float16,float16,0,0.01458666721979777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,8,128,1,float16,fp8,0,0.016336000214020412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,24,128,1,float16,float16,0,0.012938667088747025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,24,128,1,float16,fp8,0,0.013536000003417334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,1,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,1,128,1,float16,fp8,0,0.015967999895413715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,2,128,1,float16,float16,0,0.01414399966597557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,4,128,1,float16,float16,0,0.014053333550691605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,2,128,1,float16,fp8,0,0.01600533351302147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,4,128,1,float16,fp8,0,0.016143999993801117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,8,128,1,float16,float16,0,0.014090667168299357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,8,128,1,float16,fp8,0,0.016127999871969223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,1,128,1,float16,float16,0,0.05852800110975901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,1,128,1,float16,fp8,0,0.06526400148868561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,2,128,1,float16,float16,0,0.07527466615041097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,2,128,1,float16,fp8,0,0.08321600159009297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,4,128,1,float16,fp8,0,0.10584533214569092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,24,128,1,float16,float16,0,0.11762666702270508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,4,128,1,float16,float16,0,0.0922933320204417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,8,128,1,float16,float16,0,0.10603200395901997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,24,128,1,float16,fp8,0,0.11472533146540324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,1,128,1,float16,float16,0,0.028581333657105763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,8,128,1,float16,fp8,0,0.10635733604431152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,2,128,1,float16,fp8,0,0.04098666707674662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,1,128,1,float16,fp8,0,0.03311999887228012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,2,128,1,float16,float16,0,0.03578133384386698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,4,128,1,float16,float16,0,0.04460266729195913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,4,128,1,float16,fp8,0,0.04877866804599762
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,8,128,1,float16,float16,0,0.05842666824658712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,8,128,1,float16,fp8,0,0.0524479995171229
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,24,128,1,float16,float16,0,0.0614026685555776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,24,128,1,float16,fp8,0,0.056421334544817604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,1,128,1,float16,float16,0,0.02094399929046631
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,1,128,1,float16,fp8,0,0.024090667565663654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,2,128,1,float16,float16,0,0.02189333240191142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,2,128,1,float16,fp8,0,0.02566933383544286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,4,128,1,float16,float16,0,0.024480000138282776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,4,128,1,float16,fp8,0,0.029103999336560566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,8,128,1,float16,fp8,0,0.029440000653266907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,8,128,1,float16,float16,0,0.02777066578467687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,24,128,1,float16,float16,0,0.03194666653871536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,24,128,1,float16,fp8,0,0.032405334214369454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,1,128,1,float16,float16,0,0.015247999380032221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,1,128,1,float16,fp8,0,0.018112000077962875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,2,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,2,128,1,float16,fp8,0,0.018138666947682697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,4,128,1,float16,float16,0,0.01685333376129468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,24,128,1,float16,fp8,0,0.02107733239730199
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,24,128,1,float16,float16,0,0.02067199970285098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,4,128,1,float16,fp8,0,0.019567999988794327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,8,128,1,float16,float16,0,0.018496000518401463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,8,128,1,float16,fp8,0,0.01918399954835574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,1,128,1,float16,float16,0,0.015423999478419622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,1,128,1,float16,fp8,0,0.017749333133300144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,2,128,1,float16,float16,0,0.01509333277742068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,2,128,1,float16,fp8,0,0.017850667238235474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,4,128,1,float16,float16,0,0.014938666174809137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,4,128,1,float16,fp8,0,0.017840000490347546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,8,128,1,float16,float16,0,0.013616000612576803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,8,128,1,float16,fp8,0,0.013962666193644205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,24,128,1,float16,float16,0,0.016623999923467636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,24,128,1,float16,fp8,0,0.017375999440749485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,1,128,1,float16,float16,0,0.014896000425020853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,1,128,1,float16,fp8,0,0.017583999782800674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,2,128,1,float16,float16,0,0.014794666320085526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,2,128,1,float16,fp8,0,0.017866666118303936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,4,128,1,float16,float16,0,0.014720000326633453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,4,128,1,float16,fp8,0,0.017616000026464462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,8,128,1,float16,float16,0,0.01333333303531011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,8,128,1,float16,fp8,0,0.013882666826248169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,24,128,1,float16,float16,0,0.013253333667914072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,24,128,1,float16,fp8,0,0.013669333110253016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,1,128,1,float16,float16,0,0.014074667046467463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,1,128,1,float16,fp8,0,0.016154666741689045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,2,128,1,float16,float16,0,0.014389333625634512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,2,128,1,float16,fp8,0,0.016437333077192307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,4,128,1,float16,fp8,0,0.016282666474580765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,8,128,1,float16,float16,0,0.012954667210578918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,8,128,1,float16,fp8,0,0.013552000125249227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,24,128,1,float16,float16,0,0.013093333691358566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,24,128,1,float16,fp8,0,0.013936000565687815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,1,128,1,float16,float16,0,0.013951999445756277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,1,128,1,float16,fp8,0,0.015935999651749928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,2,128,1,float16,float16,0,0.015253332753976187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,2,128,1,float16,fp8,0,0.01591466615597407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,4,128,1,float16,float16,0,0.013855999956528345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,4,128,1,float16,fp8,0,0.01613866661985715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,8,128,1,float16,float16,0,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,8,128,1,float16,fp8,0,0.013253333667914072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,24,128,1,float16,float16,0,0.013056000073750814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,24,128,1,float16,fp8,0,0.013440000514189402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,1,128,1,float16,float16,0,0.013642666240533194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,1,128,1,float16,fp8,0,0.01607999950647354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,2,128,1,float16,float16,0,0.013728000223636627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,2,128,1,float16,fp8,0,0.016016000260909397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,4,128,1,float16,float16,0,0.01370666672786077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,4,128,1,float16,fp8,0,0.01591466615597407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,8,128,1,float16,float16,0,0.012304000556468964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,8,128,1,float16,fp8,0,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,1,128,1,float16,fp8,0,13.779195149739584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,1,128,1,float16,float16,0,15.205242156982422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,2,128,1,float16,float16,0,14.875855763753256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,2,128,1,float16,fp8,0,13.998565673828125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,4,128,1,float16,fp8,0,14.699509938557943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,8,128,1,float16,fp8,0,14.6125119527181
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,8,128,1,float16,float16,0,15.967461903889975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,4,128,1,float16,float16,0,16.9803466796875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,16,128,1,float16,float16,0,8.16103998819987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,1,128,1,float16,fp8,0,6.903120040893555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,1,128,1,float16,float16,0,7.5018666585286455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,16,128,1,float16,fp8,0,7.746346791585286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,2,128,1,float16,float16,0,7.378186543782552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,2,128,1,float16,fp8,0,7.054245630900065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,4,128,1,float16,float16,0,7.693871815999349
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,4,128,1,float16,fp8,0,7.435967763264974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,16,128,1,float16,float16,0,3.9717280069986978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,16,128,1,float16,fp8,0,3.810858726501465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,1,128,1,float16,float16,0,3.6225226720174155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,1,128,1,float16,fp8,0,3.5379788080851235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,2,128,1,float16,float16,0,3.595696131388346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,2,128,1,float16,fp8,0,3.5831734339396157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,8,128,1,float16,fp8,0,7.361728032430013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,8,128,1,float16,float16,0,8.058586756388346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,4,128,1,float16,float16,0,3.818431854248047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,16,128,1,float16,float16,0,2.0114506085713706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,4,128,1,float16,fp8,0,3.7477172215779624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,8,128,1,float16,float16,0,3.8632214864095054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,16,128,1,float16,fp8,0,2.0038612683614097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,8,128,1,float16,fp8,0,3.799040158589681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,1,128,1,float16,float16,0,1.8107520739237468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,1,128,1,float16,fp8,0,1.983045260111491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,2,128,1,float16,float16,0,1.8877439498901367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,2,128,1,float16,fp8,0,1.9493494033813477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,4,128,1,float16,float16,0,1.9329546292622883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,4,128,1,float16,fp8,0,2.1451093355814614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,8,128,1,float16,float16,0,1.9882666269938152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,8,128,1,float16,fp8,0,1.9555733998616536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,1,128,1,float16,fp8,0,7.927290598551433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,1,128,1,float16,float16,0,8.928266525268555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,2,128,1,float16,float16,0,8.609941482543945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,2,128,1,float16,fp8,0,8.225605646769205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,4,128,1,float16,float16,0,9.560053507486979
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,4,128,1,float16,fp8,0,8.64140256245931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,8,128,1,float16,fp8,0,8.661610921223959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,8,128,1,float16,float16,0,9.941109339396158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,16,128,1,float16,float16,0,4.609242757161458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,1,128,1,float16,fp8,0,4.011775970458984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,1,128,1,float16,float16,0,4.335455894470215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,16,128,1,float16,fp8,0,4.692970593770345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,2,128,1,float16,float16,0,4.160181363423665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,2,128,1,float16,fp8,0,4.071290651957194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,4,128,1,float16,float16,0,4.671813329060872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,4,128,1,float16,fp8,0,4.423232078552246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,16,128,1,float16,float16,0,2.3167200088500977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,16,128,1,float16,fp8,0,2.294480005900065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,1,128,1,float16,float16,0,2.099653402964274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,1,128,1,float16,fp8,0,2.0508480072021484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,8,128,1,float16,float16,0,4.708442687988281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,8,128,1,float16,fp8,0,4.357349395751953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,2,128,1,float16,float16,0,2.0971412658691406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,2,128,1,float16,fp8,0,2.0982453028361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,4,128,1,float16,float16,0,2.22380797068278
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,4,128,1,float16,fp8,0,2.2380852699279785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,16,128,1,float16,float16,0,1.2490453720092773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,8,128,1,float16,float16,0,2.2585280736287436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,16,128,1,float16,fp8,0,1.3373279571533203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,8,128,1,float16,fp8,0,2.232800006866455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,1,128,1,float16,float16,0,1.087125301361084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,1,128,1,float16,fp8,0,1.0991573333740234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,2,128,1,float16,float16,0,1.0493706862131755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,4,128,1,float16,float16,0,1.1531306902567546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,2,128,1,float16,fp8,0,1.2277493476867676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,4,128,1,float16,fp8,0,1.2050453027089436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,8,128,1,float16,float16,0,1.1640160083770752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,8,128,1,float16,fp8,0,1.215231974919637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,1,128,1,float16,float16,0,5.891504287719727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,1,128,1,float16,fp8,0,5.718778610229492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,2,128,1,float16,float16,0,6.201087951660156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,2,128,1,float16,fp8,0,5.827189127604167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,4,128,1,float16,float16,0,6.570181528727214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,4,128,1,float16,fp8,0,6.120405197143555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,8,128,1,float16,float16,0,6.8455251057942705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,16,128,1,float16,float16,0,3.2306346893310547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,1,128,1,float16,float16,0,2.875290552775065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,16,128,1,float16,fp8,0,3.6568425496419272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,1,128,1,float16,fp8,0,3.0049918492635093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,2,128,1,float16,float16,0,2.880762736002604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,2,128,1,float16,fp8,0,3.1100854873657227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,8,128,1,float16,fp8,0,6.558176040649414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,4,128,1,float16,float16,0,3.242560068766276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,16,128,1,float16,float16,0,1.6792640686035156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,4,128,1,float16,fp8,0,3.209850629170736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,16,128,1,float16,fp8,0,1.8489119211832683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,8,128,1,float16,float16,0,3.1302080154418945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,1,128,1,float16,float16,0,1.491498629252116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,1,128,1,float16,fp8,0,1.4672053654988606
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,2,128,1,float16,float16,0,1.4936480522155762
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,2,128,1,float16,fp8,0,1.5512693723042805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,4,128,1,float16,float16,0,1.6098186175028484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,4,128,1,float16,fp8,0,1.679093360900879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,16,128,1,float16,float16,0,0.8668320178985596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,8,128,1,float16,float16,0,1.6209653218587239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,8,128,1,float16,fp8,0,3.172698656717936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,16,128,1,float16,fp8,0,0.9237013657887777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,8,128,1,float16,fp8,0,1.674714724222819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,1,128,1,float16,float16,0,0.7698773543039957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,1,128,1,float16,fp8,0,0.8098399639129639
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,2,128,1,float16,float16,0,0.7692960103352865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,2,128,1,float16,fp8,0,0.7911252975463867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,4,128,1,float16,float16,0,0.8752853075663248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,8,128,1,float16,fp8,0,0.8900799751281738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,4,128,1,float16,fp8,0,0.8562400341033936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,8,128,1,float16,float16,0,0.8466560045878092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,1,128,1,float16,float16,0,7.778266906738281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,1,128,1,float16,fp8,0,7.459440231323242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,2,128,1,float16,float16,0,8.05305035909017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,2,128,1,float16,fp8,0,7.598864237467448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,4,128,1,float16,float16,0,8.731653213500977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,4,128,1,float16,fp8,0,8.325749079386393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,8,128,1,float16,float16,0,8.53544553120931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,8,128,1,float16,fp8,0,8.386394500732422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,16,128,1,float16,float16,0,4.511733373006185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,16,128,1,float16,fp8,0,4.301637331644694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,1,128,1,float16,float16,0,3.9582239786783853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,1,128,1,float16,fp8,0,3.692021369934082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,2,128,1,float16,float16,0,3.8884480794270835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,2,128,1,float16,fp8,0,3.7687625885009766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,4,128,1,float16,float16,0,4.146725336710612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,4,128,1,float16,fp8,0,4.166021347045898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,16,128,1,float16,float16,0,2.206021308898926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,16,128,1,float16,fp8,0,2.1829066276550293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,1,128,1,float16,float16,0,1.9567039807637532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,1,128,1,float16,fp8,0,1.874106725056966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,8,128,1,float16,float16,0,4.352704048156738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,8,128,1,float16,fp8,0,4.160714785257976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,2,128,1,float16,float16,0,1.8865973154703777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,2,128,1,float16,fp8,0,1.9150452613830566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,4,128,1,float16,float16,0,2.119157314300537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,4,128,1,float16,fp8,0,2.10426664352417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,16,128,1,float16,float16,0,1.1571146647135417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,16,128,1,float16,fp8,0,1.1467466354370117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,8,128,1,float16,float16,0,2.121877352396647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,2,128,1,float16,float16,0,1.0082293351491292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,1,128,1,float16,fp8,0,0.9706453482309977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,2,128,1,float16,fp8,0,0.9939520359039307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,1,128,1,float16,float16,0,1.0167786280314128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,4,128,1,float16,float16,0,1.0868693192799885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,4,128,1,float16,fp8,0,1.0932213465372722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,8,128,1,float16,fp8,0,2.1247520446777344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,16,128,1,float16,float16,0,0.5877653360366821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,8,128,1,float16,float16,0,1.09552001953125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,1,128,1,float16,fp8,0,0.5396586656570435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,8,128,1,float16,fp8,0,1.2261706988016765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,16,128,1,float16,fp8,0,0.6106826861699423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,1,128,1,float16,float16,0,0.5135466655095419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,2,128,1,float16,float16,0,0.5140479803085327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,2,128,1,float16,fp8,0,0.5362293322881063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,4,128,1,float16,float16,0,0.5521173477172852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,4,128,1,float16,fp8,0,0.6067466735839844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,8,128,1,float16,float16,0,0.5849866469701132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,8,128,1,float16,fp8,0,0.6187146504720052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,1,128,1,float16,float16,0,4.273877461751302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,1,128,1,float16,fp8,0,4.368485450744629
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,2,128,1,float16,float16,0,4.325887997945149
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,2,128,1,float16,fp8,0,4.365887959798177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,4,128,1,float16,float16,0,5.005872090657552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,4,128,1,float16,fp8,0,5.0040693283081055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,8,128,1,float16,float16,0,5.05622927347819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,16,128,1,float16,float16,0,2.612768014272054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,1,128,1,float16,float16,0,2.189904054005941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,16,128,1,float16,fp8,0,2.819823900858561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,1,128,1,float16,fp8,0,2.2508586247762046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,2,128,1,float16,float16,0,2.199962615966797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,2,128,1,float16,fp8,0,2.4417813618977866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,8,128,1,float16,fp8,0,5.421509424845378
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,4,128,1,float16,float16,0,2.4801066716512046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,16,128,1,float16,float16,0,1.3421600659688313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,4,128,1,float16,fp8,0,2.5887093544006348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,8,128,1,float16,float16,0,2.541429360707601
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,1,128,1,float16,float16,0,1.0992426872253418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,16,128,1,float16,fp8,0,1.421338717142741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,8,128,1,float16,fp8,0,2.633519967397054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,1,128,1,float16,fp8,0,1.1606187025705974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,2,128,1,float16,float16,0,1.1320266723632812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,2,128,1,float16,fp8,0,1.1739947001139324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,4,128,1,float16,fp8,0,1.3062880039215088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,4,128,1,float16,float16,0,1.2900319894154866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,8,128,1,float16,float16,0,1.293936014175415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,16,128,1,float16,fp8,0,0.7267733414967855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,8,128,1,float16,fp8,0,1.3145866394042969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,1,128,1,float16,float16,0,0.5754559834798177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,16,128,1,float16,float16,0,0.6960533459981283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,1,128,1,float16,fp8,0,0.6023946603139242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,2,128,1,float16,float16,0,0.5875413417816162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,2,128,1,float16,fp8,0,0.6331199804941813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,4,128,1,float16,float16,0,0.6609119971593221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,4,128,1,float16,fp8,0,0.6991413434346517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,8,128,1,float16,float16,0,0.6604959964752197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,8,128,1,float16,fp8,0,0.7129013538360596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,16,128,1,float16,float16,0,0.37804265817006427
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,16,128,1,float16,fp8,0,0.3953973452250163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,1,128,1,float16,float16,0,0.30856533845265705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,1,128,1,float16,fp8,0,0.3301706711451213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,2,128,1,float16,float16,0,0.3158400058746338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,2,128,1,float16,fp8,0,0.3382879892985026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,4,128,1,float16,float16,0,0.3573226531346639
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,4,128,1,float16,fp8,0,0.38233598073323566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,8,128,1,float16,float16,0,0.3653706709543864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,8,128,1,float16,fp8,0,0.38497066497802734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,1,128,1,float16,float16,0,4.114063898722331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,1,128,1,float16,fp8,0,4.164661407470703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,2,128,1,float16,float16,0,4.188207944234212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,2,128,1,float16,fp8,0,4.211861292521159
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,4,128,1,float16,float16,0,4.8115841547648115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,4,128,1,float16,fp8,0,5.0484161376953125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,8,128,1,float16,float16,0,4.9550933837890625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,8,128,1,float16,fp8,0,5.03223991394043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,1,128,1,float16,float16,0,2.038853327433268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,16,128,1,float16,float16,0,2.763514518737793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,1,128,1,float16,fp8,0,2.0672426223754883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,16,128,1,float16,fp8,0,2.673664093017578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,2,128,1,float16,float16,0,2.0993547439575195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,2,128,1,float16,fp8,0,2.131920019785563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,4,128,1,float16,float16,0,2.445919990539551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,4,128,1,float16,fp8,0,2.547050635019938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,16,128,1,float16,float16,0,1.323093334833781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,16,128,1,float16,fp8,0,1.3493226369222004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,8,128,1,float16,fp8,0,2.6618399620056152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,2,128,1,float16,float16,0,1.073413372039795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,1,128,1,float16,float16,0,1.0935839811960857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,8,128,1,float16,float16,0,2.5153013865152993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,1,128,1,float16,fp8,0,1.0599359671274822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,2,128,1,float16,fp8,0,1.0869226455688477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,4,128,1,float16,float16,0,1.242410659790039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,4,128,1,float16,fp8,0,1.2966933250427246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,8,128,1,float16,float16,0,1.2786986827850342
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,16,128,1,float16,float16,0,0.6860533555348715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,16,128,1,float16,fp8,0,0.7401333649953207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,8,128,1,float16,fp8,0,1.3634559313456218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,1,128,1,float16,float16,0,0.5283306837081909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,1,128,1,float16,fp8,0,0.551962653795878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,2,128,1,float16,float16,0,0.5501546859741211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,2,128,1,float16,fp8,0,0.5681866804758707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,4,128,1,float16,float16,0,0.6408266623814901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,4,128,1,float16,fp8,0,0.7058506806691488
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,8,128,1,float16,float16,0,0.6563306649525961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,8,128,1,float16,fp8,0,0.7062506675720215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,16,128,1,float16,float16,0,0.36213334401448566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,16,128,1,float16,fp8,0,0.384768009185791
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,1,128,1,float16,float16,0,0.28540800015131634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,1,128,1,float16,fp8,0,0.29944533109664917
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,2,128,1,float16,float16,0,0.29822399218877155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,2,128,1,float16,fp8,0,0.3168639938036601
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,4,128,1,float16,float16,0,0.3439466555913289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,4,128,1,float16,fp8,0,0.37068267663319904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,8,128,1,float16,float16,0,0.3542773326237996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,8,128,1,float16,fp8,0,0.37277865409851074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,16,128,1,float16,float16,0,0.20944533745447794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,16,128,1,float16,fp8,0,0.22055999437967935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,1,128,1,float16,float16,0,0.15986667076746622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,1,128,1,float16,fp8,0,0.16884267330169678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,2,128,1,float16,float16,0,0.16165866454442343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,2,128,1,float16,fp8,0,0.17441066106160483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,4,128,1,float16,float16,0,0.19449067115783691
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,4,128,1,float16,fp8,0,0.20141865809758505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,8,128,1,float16,float16,0,0.19889066616694132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,8,128,1,float16,fp8,0,0.21654399236043295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,1,128,1,float16,float16,0,2.3595199584960938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,1,128,1,float16,fp8,0,2.5228212674458823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,2,128,1,float16,float16,0,2.475466728210449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,2,128,1,float16,fp8,0,2.582554658253988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,4,128,1,float16,float16,0,3.053680102030436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,4,128,1,float16,fp8,0,3.1783361434936523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,8,128,1,float16,float16,0,3.072175979614258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,1,128,1,float16,float16,0,1.2094293435414631
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,16,128,1,float16,float16,0,1.6679040590922039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,16,128,1,float16,fp8,0,1.8498506546020508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,1,128,1,float16,fp8,0,1.2584959665934246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,2,128,1,float16,float16,0,1.2569759686787922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,8,128,1,float16,fp8,0,3.318661371866862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,2,128,1,float16,fp8,0,1.3211466471354167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,4,128,1,float16,float16,0,1.5272053082784016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,4,128,1,float16,fp8,0,1.6482666333516438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,16,128,1,float16,float16,0,0.8410346508026123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,8,128,1,float16,float16,0,1.5611732800801594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,1,128,1,float16,float16,0,0.6212960084279379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,16,128,1,float16,fp8,0,0.8994932969411215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,8,128,1,float16,fp8,0,1.6688586870829265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,1,128,1,float16,fp8,0,0.6549599965413412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,2,128,1,float16,float16,0,0.6619093418121338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,2,128,1,float16,fp8,0,0.6760266621907552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,4,128,1,float16,float16,0,0.779034694035848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,4,128,1,float16,fp8,0,0.8475946585337321
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,8,128,1,float16,float16,0,0.787162701288859
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,16,128,1,float16,float16,0,0.44340264797210693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,16,128,1,float16,fp8,0,0.47709333896636963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,8,128,1,float16,fp8,0,0.8611893653869629
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,1,128,1,float16,float16,0,0.32763199011484784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,2,128,1,float16,float16,0,0.35019731521606445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,1,128,1,float16,fp8,0,0.35204267501831055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,2,128,1,float16,fp8,0,0.3601919809977214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,4,128,1,float16,float16,0,0.4106666644414266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,4,128,1,float16,fp8,0,0.44758931795756024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,8,128,1,float16,float16,0,0.4217439889907837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,1,128,1,float16,fp8,0,0.1934880018234253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,8,128,1,float16,fp8,0,0.4577920039494832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,16,128,1,float16,float16,0,0.24041066567103067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,16,128,1,float16,fp8,0,0.2618880073229472
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,1,128,1,float16,float16,0,0.1819253365198771
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,2,128,1,float16,float16,0,0.18929600715637207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,2,128,1,float16,fp8,0,0.1998293399810791
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,4,128,1,float16,float16,0,0.22615466515223184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,4,128,1,float16,fp8,0,0.24116265773773193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,1,128,1,float16,float16,0,0.10110400120417277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,8,128,1,float16,float16,0,0.2315679987271627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,8,128,1,float16,fp8,0,0.253493328889211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,2,128,1,float16,fp8,0,0.11518933375676473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,16,128,1,float16,float16,0,0.13966932892799377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,16,128,1,float16,fp8,0,0.1553920010725657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,1,128,1,float16,fp8,0,0.10920533537864685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,2,128,1,float16,float16,0,0.10637866457303365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,4,128,1,float16,float16,0,0.12641066312789917
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,4,128,1,float16,fp8,0,0.1307253340880076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,8,128,1,float16,float16,0,0.1330933372179667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,8,128,1,float16,fp8,0,0.14495999614397684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,1,128,1,float16,float16,0,2.3380746841430664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,1,128,1,float16,fp8,0,2.520655949910482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,2,128,1,float16,float16,0,2.550970713297526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,2,128,1,float16,fp8,0,2.593503952026367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,4,128,1,float16,float16,0,3.1850719451904297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,4,128,1,float16,fp8,0,3.4557387034098306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,8,128,1,float16,float16,0,3.217701276143392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,16,128,1,float16,float16,0,1.7795626322428386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,8,128,1,float16,fp8,0,3.4290507634480796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,16,128,1,float16,fp8,0,1.8498133023579915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,1,128,1,float16,float16,0,1.187013308207194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,1,128,1,float16,fp8,0,1.2686186631520588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,2,128,1,float16,float16,0,1.267317295074463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,2,128,1,float16,fp8,0,1.3139946460723877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,4,128,1,float16,float16,0,1.6098079681396484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,4,128,1,float16,fp8,0,1.7451093991597493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,8,128,1,float16,float16,0,1.615872065226237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,16,128,1,float16,float16,0,0.904416004816691
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,16,128,1,float16,fp8,0,0.9296000003814697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,8,128,1,float16,fp8,0,1.7227253913879395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,1,128,1,float16,float16,0,0.6058826843897501
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,1,128,1,float16,fp8,0,0.6467413504918417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,2,128,1,float16,float16,0,0.6587946812311808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,4,128,1,float16,fp8,0,0.904047966003418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,2,128,1,float16,fp8,0,0.6809653441111246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,4,128,1,float16,float16,0,0.8165866533915201
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,8,128,1,float16,float16,0,0.841439962387085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,16,128,1,float16,float16,0,0.4669813315073649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,8,128,1,float16,fp8,0,0.9130079746246338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,16,128,1,float16,fp8,0,0.5022986729939779
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,1,128,1,float16,float16,0,0.3168426752090454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,1,128,1,float16,fp8,0,0.34876267115275067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,2,128,1,float16,float16,0,0.34654398759206134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,2,128,1,float16,fp8,0,0.3609600067138672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,8,128,1,float16,fp8,0,0.4737120072046916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,4,128,1,float16,float16,0,0.4245866537094116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,4,128,1,float16,fp8,0,0.46989333629608154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,8,128,1,float16,float16,0,0.42952533562978107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,16,128,1,float16,float16,0,0.2516000072161357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,2,128,1,float16,float16,0,0.18797866503397623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,16,128,1,float16,fp8,0,0.2744106650352478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,1,128,1,float16,float16,0,0.1747679909070333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,4,128,1,float16,fp8,0,0.25173334280649823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,1,128,1,float16,fp8,0,0.19023466110229492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,2,128,1,float16,fp8,0,0.20012799898783365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,4,128,1,float16,float16,0,0.23113600413004556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,8,128,1,float16,float16,0,0.23527467250823975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,8,128,1,float16,fp8,0,0.2635733286539714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,16,128,1,float16,float16,0,0.14013866583506265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,16,128,1,float16,fp8,0,0.15275733669598898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,1,128,1,float16,float16,0,0.09709333380063374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,1,128,1,float16,fp8,0,0.10697600245475769
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,2,128,1,float16,float16,0,0.09970133503278096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,2,128,1,float16,fp8,0,0.10824533303578694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,4,128,1,float16,float16,0,0.12947733203570047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,4,128,1,float16,fp8,0,0.13984533150990805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,8,128,1,float16,float16,0,0.13291733463605246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,8,128,1,float16,fp8,0,0.14607466260592142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,16,128,1,float16,float16,0,0.08399466673533122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,16,128,1,float16,fp8,0,0.09139733513196309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,1,128,1,float16,float16,0,0.05612266560395559
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,1,128,1,float16,fp8,0,0.06287999947865804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,2,128,1,float16,float16,0,0.0620000014702479
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,2,128,1,float16,fp8,0,0.0702400008837382
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,4,128,1,float16,float16,0,0.06989866495132446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,4,128,1,float16,fp8,0,0.0784746656815211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,8,128,1,float16,float16,0,0.07458133498827617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,8,128,1,float16,fp8,0,0.07987200220425923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,1,128,1,float16,float16,0,1.4483092625935872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,1,128,1,float16,fp8,0,1.5677706400553386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,2,128,1,float16,float16,0,1.5340159734090169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,2,128,1,float16,fp8,0,1.627855936686198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,4,128,1,float16,float16,0,2.086730639139811
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,8,128,1,float16,float16,0,2.1253387133280435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,4,128,1,float16,fp8,0,2.284693400065104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,16,128,1,float16,float16,0,1.1777439912160237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,16,128,1,float16,fp8,0,1.284058650334676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,8,128,1,float16,fp8,0,2.354053338368734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,1,128,1,float16,fp8,0,0.8066559632619222
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,1,128,1,float16,float16,0,0.7443679968516032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,2,128,1,float16,float16,0,0.8085386753082275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,4,128,1,float16,float16,0,1.0516640345255535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,2,128,1,float16,fp8,0,0.8346827030181885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,4,128,1,float16,fp8,0,1.1717759768168132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,8,128,1,float16,float16,0,1.0834879875183105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,16,128,1,float16,float16,0,0.6034239927927653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,1,128,1,float16,float16,0,0.3850613435109456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,8,128,1,float16,fp8,0,1.19595201810201
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,2,128,1,float16,float16,0,0.4211466709772746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,2,128,1,float16,fp8,0,0.4421759843826294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,1,128,1,float16,fp8,0,0.41656001408894855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,16,128,1,float16,fp8,0,0.6590773264567057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,8,128,1,float16,float16,0,0.5568000078201294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,4,128,1,float16,fp8,0,0.5994186798731486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,16,128,1,float16,float16,0,0.31563733021418255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,8,128,1,float16,fp8,0,0.6242026487986246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,4,128,1,float16,float16,0,0.5485173463821411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,16,128,1,float16,fp8,0,0.3460906744003296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,1,128,1,float16,float16,0,0.20429333051045737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,4,128,1,float16,float16,0,0.2906506657600403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,1,128,1,float16,fp8,0,0.22454400857289633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,2,128,1,float16,float16,0,0.22904000679651895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,2,128,1,float16,fp8,0,0.23662400245666504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,4,128,1,float16,fp8,0,0.3186560074488322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,8,128,1,float16,float16,0,0.29665066798528034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,8,128,1,float16,fp8,0,0.3301226695378621
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,2,128,1,float16,float16,0,0.12627733747164407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,16,128,1,float16,float16,0,0.1766080061594645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,16,128,1,float16,fp8,0,0.19249065717061362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,1,128,1,float16,float16,0,0.11763733625411987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,1,128,1,float16,fp8,0,0.1279093325138092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,2,128,1,float16,fp8,0,0.1325706640879313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,4,128,1,float16,float16,0,0.15519466996192932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,4,128,1,float16,fp8,0,0.17410133282343546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,8,128,1,float16,float16,0,0.16312000155448914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,8,128,1,float16,fp8,0,0.1799359917640686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,16,128,1,float16,float16,0,0.09864532947540283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,16,128,1,float16,fp8,0,0.11333866914113362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,1,128,1,float16,float16,0,0.06564799944559734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,1,128,1,float16,fp8,0,0.0714026689529419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,2,128,1,float16,float16,0,0.0690773328145345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,2,128,1,float16,fp8,0,0.07772799829641978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,4,128,1,float16,float16,0,0.08627733588218689
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,4,128,1,float16,fp8,0,0.089519997437795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,8,128,1,float16,float16,0,0.09370666742324829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,8,128,1,float16,fp8,0,0.10367467006047566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,16,128,1,float16,float16,0,0.05927466849486033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,16,128,1,float16,fp8,0,0.06555733581384023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,1,128,1,float16,float16,0,0.04320533573627472
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,1,128,1,float16,fp8,0,0.04872000217437744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,2,128,1,float16,float16,0,0.0439626673857371
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,2,128,1,float16,fp8,0,0.049413333336512245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,4,128,1,float16,float16,0,0.05019199848175049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,4,128,1,float16,fp8,0,0.057034666339556374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,8,128,1,float16,float16,0,0.05120000243186951
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,8,128,1,float16,fp8,0,0.05876266459623972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,1,128,1,float16,float16,0,1.5311147371927898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,1,128,1,float16,fp8,0,1.6978665987650554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,2,128,1,float16,float16,0,1.7205440203348796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,8,128,1,float16,float16,0,2.410154660542806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,2,128,1,float16,fp8,0,1.7681387265523274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,16,128,1,float16,float16,0,1.350005308787028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,4,128,1,float16,float16,0,2.3416159947713218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,4,128,1,float16,fp8,0,2.6527679761250815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,1,128,1,float16,float16,0,0.7855573495229086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,16,128,1,float16,fp8,0,1.5063999493916829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,1,128,1,float16,fp8,0,0.8599519729614258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,2,128,1,float16,float16,0,0.8658826351165771
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,8,128,1,float16,fp8,0,2.734656016031901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,8,128,1,float16,float16,0,1.2153600056966145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,16,128,1,float16,float16,0,0.6933386325836182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,2,128,1,float16,fp8,0,0.8988107045491537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,16,128,1,float16,fp8,0,0.7466399669647217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,4,128,1,float16,float16,0,1.2009812990824382
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,4,128,1,float16,fp8,0,1.3489386240641277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,8,128,1,float16,fp8,0,1.3471892674763997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,1,128,1,float16,float16,0,0.4082346757253011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,1,128,1,float16,fp8,0,0.44857064882914227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,2,128,1,float16,float16,0,0.4548213481903076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,2,128,1,float16,fp8,0,0.46952001253763836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,4,128,1,float16,float16,0,0.6069226662317911
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,4,128,1,float16,fp8,0,0.68941863377889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,8,128,1,float16,float16,0,0.6327626705169678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,16,128,1,float16,float16,0,0.362277348836263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,8,128,1,float16,fp8,0,0.7186079819997152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,16,128,1,float16,fp8,0,0.4033706585566203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,1,128,1,float16,float16,0,0.2129439910252889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,1,128,1,float16,fp8,0,0.23614400625228882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,8,128,1,float16,float16,0,0.33218133449554443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,8,128,1,float16,fp8,0,0.3717120091120402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,2,128,1,float16,fp8,0,0.2509066661198934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,16,128,1,float16,fp8,0,0.21454399824142456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,16,128,1,float16,float16,0,0.19056532780329385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,2,128,1,float16,float16,0,0.24217599630355835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,2,128,1,float16,float16,0,0.13217066725095114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,4,128,1,float16,float16,0,0.31913065910339355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,4,128,1,float16,fp8,0,0.35736000537872314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,1,128,1,float16,fp8,0,0.13217066725095114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,1,128,1,float16,float16,0,0.12332800030708313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,2,128,1,float16,fp8,0,0.1397279997666677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,4,128,1,float16,float16,0,0.17073599497477213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,4,128,1,float16,fp8,0,0.19402132431666055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,8,128,1,float16,float16,0,0.17891200383504233
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,8,128,1,float16,fp8,0,0.19940799474716187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,16,128,1,float16,float16,0,0.10622933506965637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,16,128,1,float16,fp8,0,0.124208003282547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,1,128,1,float16,float16,0,0.06901333232720692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,1,128,1,float16,fp8,0,0.07423466444015503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,8,128,1,float16,float16,0,0.09860799709955852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,2,128,1,float16,float16,0,0.06912533442179362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,8,128,1,float16,fp8,0,0.11414933204650879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,2,128,1,float16,fp8,0,0.07878399888674419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,4,128,1,float16,float16,0,0.09596266349156697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,16,128,1,float16,fp8,0,0.07144000132878621
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,4,128,1,float16,fp8,0,0.10586667060852051
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,1,128,1,float16,float16,0,0.04081066697835922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,16,128,1,float16,float16,0,0.0645066648721695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,2,128,1,float16,fp8,0,0.05119466781616211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,1,128,1,float16,fp8,0,0.04674133161703745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,2,128,1,float16,float16,0,0.04372266431649526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,4,128,1,float16,float16,0,0.0506933331489563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,8,128,1,float16,fp8,0,0.05893866717815399
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,4,128,1,float16,fp8,0,0.057205334305763245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,8,128,1,float16,float16,0,0.0547680010398229
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,16,128,1,float16,float16,0,0.03145066648721695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,1,128,1,float16,float16,0,0.026485333840052288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,16,128,1,float16,fp8,0,0.03709866603215536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,1,128,1,float16,fp8,0,0.029578665892283123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,4,128,1,float16,float16,0,0.030554667115211487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,2,128,1,float16,fp8,0,0.031061333914597828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,2,128,1,float16,float16,0,0.02731200059254964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,8,128,1,float16,float16,0,0.030778666337331135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,4,128,1,float16,fp8,0,0.0345920001467069
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,8,128,1,float16,fp8,0,0.03535466641187668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,1,128,1,float16,float16,0,1.1627093156178792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,1,128,1,float16,fp8,0,1.3090773423512776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,2,128,1,float16,float16,0,1.3232639630635579
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,2,128,1,float16,fp8,0,1.3852052688598633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,16,128,1,float16,float16,0,1.1612746715545654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,1,128,1,float16,float16,0,0.5889706611633301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,16,128,1,float16,fp8,0,1.3011306921641033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,4,128,1,float16,float16,0,1.9785866737365723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,4,128,1,float16,fp8,0,2.2795732816060386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,1,128,1,float16,fp8,0,0.6690933704376221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,2,128,1,float16,fp8,0,0.7061920166015625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,2,128,1,float16,float16,0,0.6770559946695963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,4,128,1,float16,fp8,0,1.158618688583374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,8,128,1,float16,float16,0,1.0184053579966228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,4,128,1,float16,float16,0,1.006656010945638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,16,128,1,float16,float16,0,0.5914400021235148
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,8,128,1,float16,float16,0,2.0109386444091797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,8,128,1,float16,fp8,0,1.168399969736735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,16,128,1,float16,fp8,0,0.668165365854899
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,8,128,1,float16,fp8,0,2.3230667114257812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,1,128,1,float16,float16,0,0.3051253358523051
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,2,128,1,float16,float16,0,0.3625919818878174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,2,128,1,float16,fp8,0,0.36875732739766437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,1,128,1,float16,fp8,0,0.3473653395970662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,16,128,1,float16,float16,0,0.3072426716486613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,8,128,1,float16,float16,0,0.5367733240127563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,1,128,1,float16,float16,0,0.16363733013470969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,8,128,1,float16,fp8,0,0.6325973272323608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,16,128,1,float16,fp8,0,0.364464004834493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,4,128,1,float16,fp8,0,0.5900853474934896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,4,128,1,float16,float16,0,0.5112160046895345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,1,128,1,float16,fp8,0,0.18721065918604532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,2,128,1,float16,fp8,0,0.19830399751663208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,2,128,1,float16,float16,0,0.19298134247461954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,8,128,1,float16,float16,0,0.27928000688552856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,4,128,1,float16,fp8,0,0.30877333879470825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,8,128,1,float16,fp8,0,0.32530667384465534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,16,128,1,float16,float16,0,0.16268799702326456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,16,128,1,float16,fp8,0,0.19402132431666055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,1,128,1,float16,float16,0,0.09204799930254619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,4,128,1,float16,float16,0,0.2674506704012553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,1,128,1,float16,fp8,0,0.10530666510264079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,2,128,1,float16,float16,0,0.10386666655540466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,2,128,1,float16,fp8,0,0.1104853351910909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,4,128,1,float16,float16,0,0.1434986690680186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,4,128,1,float16,fp8,0,0.1650879979133606
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,8,128,1,float16,float16,0,0.14989333351453146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,8,128,1,float16,fp8,0,0.17894933621088663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,1,128,1,float16,fp8,0,0.057146668434143066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,16,128,1,float16,float16,0,0.09180800120035808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,2,128,1,float16,float16,0,0.05385600030422211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,16,128,1,float16,fp8,0,0.11062933007876079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,1,128,1,float16,float16,0,0.05315199991067251
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,2,128,1,float16,fp8,0,0.060378665725390114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,4,128,1,float16,float16,0,0.07951466739177704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,4,128,1,float16,fp8,0,0.0909546713034312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,8,128,1,float16,float16,0,0.08333333333333333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,8,128,1,float16,fp8,0,0.10107733805974324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,16,128,1,float16,float16,0,0.052602668603261314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,16,128,1,float16,fp8,0,0.06092800199985504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,1,128,1,float16,float16,0,0.03156266609827677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,1,128,1,float16,fp8,0,0.03634133438269297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,2,128,1,float16,float16,0,0.03322133421897888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,2,128,1,float16,fp8,0,0.0386613334218661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,4,128,1,float16,float16,0,0.03884266565243403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,4,128,1,float16,fp8,0,0.045653333266576133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,8,128,1,float16,fp8,0,0.051514665285746254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,8,128,1,float16,float16,0,0.04302933315436045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,16,128,1,float16,float16,0,0.02552533398071925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,16,128,1,float16,fp8,0,0.03154666721820831
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,1,128,1,float16,float16,0,0.020303999384244282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,1,128,1,float16,fp8,0,0.023823998868465424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,2,128,1,float16,float16,0,0.022330666581789654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,2,128,1,float16,fp8,0,0.025589334468046825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,4,128,1,float16,float16,0,0.02442666639884313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,4,128,1,float16,fp8,0,0.028832000990708668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,8,128,1,float16,float16,0,0.02442666639884313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,1,128,1,float16,fp8,0,0.02342933416366577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,8,128,1,float16,fp8,0,0.030661332110563915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,16,128,1,float16,float16,0,0.02187199890613556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,4,128,1,float16,float16,0,0.02128000060717265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,4,128,1,float16,fp8,0,0.025610665480295818
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,16,128,1,float16,fp8,0,0.027215999861558277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,1,128,1,float16,float16,0,0.02042666698495547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,2,128,1,float16,float16,0,0.02035733312368393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,2,128,1,float16,fp8,0,0.023930666347344715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,8,128,1,float16,float16,0,0.021290667355060577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,8,128,1,float16,fp8,0,0.02794133375088374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,1,128,1,float16,fp8,0,0.5723413228988647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,1,128,1,float16,float16,0,0.49537599086761475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,2,128,1,float16,float16,0,0.5817973216374716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,2,128,1,float16,fp8,0,0.6106880108515421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,4,128,1,float16,float16,0,0.9166773160298666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,4,128,1,float16,fp8,0,1.0623733202616374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,16,128,1,float16,float16,0,0.5501120090484619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,16,128,1,float16,fp8,0,0.5924426714579264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,8,128,1,float16,float16,0,0.9514719645182291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,8,128,1,float16,fp8,0,1.0764959653218586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,1,128,1,float16,float16,0,0.2597653269767761
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,1,128,1,float16,fp8,0,0.29897065957387287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,16,128,1,float16,float16,0,0.2860693335533142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,16,128,1,float16,fp8,0,0.3279199997584025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,2,128,1,float16,float16,0,0.301418662071228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,2,128,1,float16,fp8,0,0.3208426634470622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,4,128,1,float16,float16,0,0.46747199694315594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,4,128,1,float16,fp8,0,0.5443253517150879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,1,128,1,float16,float16,0,0.14089600245157877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,8,128,1,float16,float16,0,0.4909919897715251
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,1,128,1,float16,fp8,0,0.16168533762296042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,8,128,1,float16,fp8,0,0.5818346738815308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,2,128,1,float16,float16,0,0.16674667596817017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,8,128,1,float16,float16,0,0.2557706634203593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,2,128,1,float16,fp8,0,0.17223467429478964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,4,128,1,float16,fp8,0,0.2841493288675944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,4,128,1,float16,float16,0,0.24225066105524698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,16,128,1,float16,float16,0,0.15269333124160767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,1,128,1,float16,float16,0,0.07924800117810567
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,8,128,1,float16,fp8,0,0.300981342792511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,2,128,1,float16,float16,0,0.08974400162696838
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,1,128,1,float16,fp8,0,0.09242133299509685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,2,128,1,float16,fp8,0,0.09823466340700786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,16,128,1,float16,fp8,0,0.17469332615534464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,4,128,1,float16,float16,0,0.13081066807111105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,4,128,1,float16,fp8,0,0.15223466356595358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,8,128,1,float16,float16,0,0.13825600345929465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,16,128,1,float16,float16,0,0.08545600374539693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,8,128,1,float16,fp8,0,0.16565333803494772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,16,128,1,float16,fp8,0,0.10053867101669312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,1,128,1,float16,float16,0,0.04446933170159658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,1,128,1,float16,fp8,0,0.04971733192602793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,2,128,1,float16,float16,0,0.04607999821503957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,2,128,1,float16,fp8,0,0.051685333251953125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,4,128,1,float16,float16,0,0.07256533205509186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,4,128,1,float16,fp8,0,0.081535999973615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,8,128,1,float16,float16,0,0.07586133480072021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,8,128,1,float16,fp8,0,0.09206933776537578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,1,128,1,float16,fp8,0,0.030346666773160298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,16,128,1,float16,float16,0,0.04730666677157084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,16,128,1,float16,fp8,0,0.05507733424504598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,1,128,1,float16,float16,0,0.025578667720158894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,2,128,1,float16,float16,0,0.027642667293548584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,2,128,1,float16,fp8,0,0.032730666299661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,8,128,1,float16,fp8,0,0.043696001172065735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,4,128,1,float16,float16,0,0.03311466674009959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,4,128,1,float16,fp8,0,0.040175999204317726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,8,128,1,float16,float16,0,0.03851199895143509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,16,128,1,float16,float16,0,0.023386667172114056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,16,128,1,float16,fp8,0,0.028255999088287354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,1,128,1,float16,float16,0,0.017370666066805523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,1,128,1,float16,fp8,0,0.020687999824682873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,2,128,1,float16,float16,0,0.01876266673207283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,2,128,1,float16,fp8,0,0.021615999440352123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,8,128,1,float16,float16,0,0.021226666867733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,16,128,1,float16,float16,0,0.01924266666173935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,8,128,1,float16,fp8,0,0.02681066592534383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,4,128,1,float16,float16,0,0.021365332106749218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,1,128,1,float16,float16,0,0.016997333616018295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,16,128,1,float16,fp8,0,0.02402133246262868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,4,128,1,float16,fp8,0,0.025413334369659424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,1,128,1,float16,fp8,0,0.01951466624935468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,8,128,1,float16,float16,0,0.01851733277241389
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,2,128,1,float16,float16,0,0.017903999735911686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,2,128,1,float16,fp8,0,0.019839999576409657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,4,128,1,float16,float16,0,0.018559999763965607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,4,128,1,float16,fp8,0,0.02186666677395503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,8,128,1,float16,fp8,0,0.02306666721900304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,16,128,1,float16,float16,0,0.01738133281469345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,16,128,1,float16,fp8,0,0.022453332940737408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,2,128,1,float16,fp8,0,0.01979200045267741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,1,128,1,float16,float16,0,0.016965333372354507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,1,128,1,float16,fp8,0,0.019066666563351948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,2,128,1,float16,float16,0,0.01703466723362605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,4,128,1,float16,float16,0,0.017338667064905167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,4,128,1,float16,fp8,0,0.019498666127522785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,8,128,1,float16,float16,0,0.01711999997496605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,8,128,1,float16,fp8,0,0.021573332448800404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,1,128,1,float16,float16,0,0.2466613252957662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,1,128,1,float16,fp8,0,0.27270932992299396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,2,128,1,float16,float16,0,0.3033173282941182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,2,128,1,float16,fp8,0,0.27989333868026733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,16,128,1,float16,float16,0,0.27711466948191327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,4,128,1,float16,fp8,0,0.5214346647262573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,1,128,1,float16,float16,0,0.13556266824404398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,16,128,1,float16,fp8,0,0.29976000388463336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,1,128,1,float16,fp8,0,0.15241600076357523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,8,128,1,float16,float16,0,0.47197333971659344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,4,128,1,float16,float16,0,0.45442132155100506
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,2,128,1,float16,float16,0,0.16166933377583823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,8,128,1,float16,fp8,0,0.5374240080515543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,4,128,1,float16,float16,0,0.23523734013239542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,2,128,1,float16,fp8,0,0.15503467122713724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,4,128,1,float16,fp8,0,0.2737119992574056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,8,128,1,float16,float16,0,0.24617600440979004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,16,128,1,float16,fp8,0,0.16180266936620077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,16,128,1,float16,float16,0,0.14778666694959006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,2,128,1,float16,float16,0,0.0844693382581075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,8,128,1,float16,fp8,0,0.2821173270543416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,2,128,1,float16,fp8,0,0.08803733189900716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,1,128,1,float16,fp8,0,0.08642666538556416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,4,128,1,float16,float16,0,0.1269706686337789
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,4,128,1,float16,fp8,0,0.1472053329149882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,8,128,1,float16,fp8,0,0.1530080040295919
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,8,128,1,float16,float16,0,0.13294399778048197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,16,128,1,float16,fp8,0,0.08959999680519104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,1,128,1,float16,float16,0,0.042277331153551735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,1,128,1,float16,fp8,0,0.04317333300908407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,16,128,1,float16,float16,0,0.08261866867542267
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,2,128,1,float16,float16,0,0.04203199843565623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,8,128,1,float16,float16,0,0.07431999842325847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,4,128,1,float16,fp8,0,0.07496533294518788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,2,128,1,float16,fp8,0,0.04508799811204275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,4,128,1,float16,float16,0,0.07050666709740956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,8,128,1,float16,fp8,0,0.08347200353940327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,16,128,1,float16,fp8,0,0.0487360010544459
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,16,128,1,float16,float16,0,0.045423999428749084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,2,128,1,float16,float16,0,0.025050667424996693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,1,128,1,float16,float16,0,0.023232000569502514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,1,128,1,float16,fp8,0,0.026357332865397137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,1,128,1,float16,float16,0,0.07654400169849396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,2,128,1,float16,fp8,0,0.02867199977238973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,8,128,1,float16,float16,0,0.03442666679620743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,8,128,1,float16,fp8,0,0.03726933399836222
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,4,128,1,float16,fp8,0,0.03594133257865906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,4,128,1,float16,float16,0,0.030799999833106995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,16,128,1,float16,float16,0,0.02109333376089732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,16,128,1,float16,fp8,0,0.02401600033044815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,1,128,1,float16,float16,0,0.01607999950647354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,2,128,1,float16,float16,0,0.017231999586025875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,2,128,1,float16,fp8,0,0.019733333339293797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,8,128,1,float16,float16,0,0.019941333681344986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,4,128,1,float16,float16,0,0.019882666567961376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,1,128,1,float16,fp8,0,0.018645333747069042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,16,128,1,float16,fp8,0,0.01987733319401741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,8,128,1,float16,fp8,0,0.023376000424226124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,4,128,1,float16,fp8,0,0.0233599990606308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,16,128,1,float16,float16,0,0.01743999992807706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,1,128,1,float16,float16,0,0.015589332828919092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,1,128,1,float16,fp8,0,0.0182239996890227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,2,128,1,float16,float16,0,0.015290666371583939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,2,128,1,float16,fp8,0,0.017973333597183228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,4,128,1,float16,float16,0,0.01699200024207433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,4,128,1,float16,fp8,0,0.019717333217461903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,8,128,1,float16,float16,0,0.01714666684468587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,8,128,1,float16,fp8,0,0.019621333728233974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,16,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,16,128,1,float16,fp8,0,0.017727999637524288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,1,128,1,float16,float16,0,0.015130666395028433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,2,128,1,float16,float16,0,0.01504533365368843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,2,128,1,float16,fp8,0,0.01721599946419398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,4,128,1,float16,float16,0,0.015381333728631338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,1,128,1,float16,fp8,0,0.01800000046690305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,8,128,1,float16,fp8,0,0.017871999492247898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,8,128,1,float16,float16,0,0.015610666324694952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,4,128,1,float16,fp8,0,0.017973333597183228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,16,128,1,float16,fp8,0,0.017583999782800674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,16,128,1,float16,float16,0,0.014592000593741735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,1,128,1,float16,float16,0,0.014864000181357065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,1,128,1,float16,fp8,0,0.016506666938463848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,2,128,1,float16,float16,0,0.014442666123310724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,4,128,1,float16,float16,0,0.01444799949725469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,2,128,1,float16,fp8,0,0.017242666333913803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,4,128,1,float16,fp8,0,0.016629333297411602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,8,128,1,float16,float16,0,0.014442666123310724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,8,128,1,float16,fp8,0,0.01661866654952367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,1,128,1,float16,float16,0,0.1335093379020691
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,1,128,1,float16,fp8,0,0.1498293379942576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,2,128,1,float16,float16,0,0.1599999964237213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,2,128,1,float16,fp8,0,0.151936004559199
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,4,128,1,float16,float16,0,0.23574932416280112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,16,128,1,float16,float16,0,0.1574560006459554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,4,128,1,float16,fp8,0,0.27453333139419556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,8,128,1,float16,float16,0,0.24263467391331991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,16,128,1,float16,fp8,0,0.15357866883277893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,1,128,1,float16,float16,0,0.07665599882602692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,8,128,1,float16,fp8,0,0.2767413258552551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,1,128,1,float16,fp8,0,0.08567466338475545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,2,128,1,float16,float16,0,0.08331199983755748
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,2,128,1,float16,fp8,0,0.08822400371233623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,4,128,1,float16,float16,0,0.12596266468365988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,4,128,1,float16,fp8,0,0.14615999658902487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,8,128,1,float16,float16,0,0.1346986691157023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,8,128,1,float16,fp8,0,0.1511679987112681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,16,128,1,float16,fp8,0,0.08543466528256734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,16,128,1,float16,float16,0,0.08745066324869792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,1,128,1,float16,float16,0,0.03958400090535482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,1,128,1,float16,fp8,0,0.04499199986457825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,2,128,1,float16,float16,0,0.040976000328858696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,2,128,1,float16,fp8,0,0.0451200008392334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,4,128,1,float16,float16,0,0.07036266724268596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,4,128,1,float16,fp8,0,0.07560533285140991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,8,128,1,float16,float16,0,0.07338133454322815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,8,128,1,float16,fp8,0,0.08398399750391643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,16,128,1,float16,float16,0,0.049770668148994446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,16,128,1,float16,fp8,0,0.04364266494909922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,1,128,1,float16,fp8,0,0.02643733223279317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,1,128,1,float16,float16,0,0.022863999009132385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,2,128,1,float16,float16,0,0.025045332809289295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,2,128,1,float16,fp8,0,0.028773332635561626
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,4,128,1,float16,float16,0,0.03030933439731598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,4,128,1,float16,fp8,0,0.03575466573238373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,1,128,1,float16,float16,0,0.016069332758585613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,8,128,1,float16,float16,0,0.03522133330504099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,8,128,1,float16,fp8,0,0.037717332442601524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,16,128,1,float16,float16,0,0.021903999149799347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,16,128,1,float16,fp8,0,0.02325333406527837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,1,128,1,float16,fp8,0,0.018735999862353008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,8,128,1,float16,float16,0,0.019727999965349834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,2,128,1,float16,float16,0,0.017157333592573803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,2,128,1,float16,fp8,0,0.019658666104078293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,4,128,1,float16,float16,0,0.019776000330845516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,4,128,1,float16,fp8,0,0.02325333406527837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,8,128,1,float16,fp8,0,0.023317334552605946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,16,128,1,float16,float16,0,0.015840000162522
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,16,128,1,float16,fp8,0,0.01613333324591319
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,1,128,1,float16,float16,0,0.015200000256299973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,1,128,1,float16,fp8,0,0.01766933376590411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,2,128,1,float16,float16,0,0.01543466622630755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,2,128,1,float16,fp8,0,0.017466666797796886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,4,128,1,float16,float16,0,0.016864000509182613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,4,128,1,float16,fp8,0,0.019589333484570186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,8,128,1,float16,float16,0,0.01706133286158244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,8,128,1,float16,fp8,0,0.019754666835069656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,16,128,1,float16,float16,0,0.013674666484196981
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,16,128,1,float16,fp8,0,0.013722666849692663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,1,128,1,float16,float16,0,0.015040000279744467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,1,128,1,float16,fp8,0,0.017194667210181553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,2,128,1,float16,float16,0,0.014752000570297241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,2,128,1,float16,fp8,0,0.017269333203633625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,4,128,1,float16,float16,0,0.01605333387851715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,4,128,1,float16,fp8,0,0.017450666675964992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,8,128,1,float16,float16,0,0.015290666371583939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,8,128,1,float16,fp8,0,0.01766933376590411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,16,128,1,float16,float16,0,0.013082666943470636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,16,128,1,float16,fp8,0,0.013477332890033722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,1,128,1,float16,float16,0,0.015050667027632395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,1,128,1,float16,fp8,0,0.016613333175579708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,8,128,1,float16,float16,0,0.014432000617186228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,2,128,1,float16,float16,0,0.014170666535695394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,2,128,1,float16,fp8,0,0.01586666703224182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,4,128,1,float16,float16,0,0.014426667243242264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,4,128,1,float16,fp8,0,0.01632000009218852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,8,128,1,float16,fp8,0,0.017477333545684814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,16,128,1,float16,float16,0,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,16,128,1,float16,fp8,0,0.012879999975363413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,1,128,1,float16,float16,0,0.014192000031471252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,1,128,1,float16,fp8,0,0.015978666643301647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,2,128,1,float16,float16,0,0.014096000542243322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,2,128,1,float16,fp8,0,0.015893333901961643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,4,128,1,float16,float16,0,0.014181333283583323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,4,128,1,float16,fp8,0,0.01632000009218852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,8,128,1,float16,float16,0,0.014287999520699183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,8,128,1,float16,fp8,0,0.016330666840076447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,1,128,1,float16,float16,0,0.07606933514277141
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,1,128,1,float16,fp8,0,0.08597333232561748
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,2,128,1,float16,float16,0,0.08243733147780101
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,2,128,1,float16,fp8,0,0.08725866675376892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,4,128,1,float16,float16,0,0.12623467048009238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,4,128,1,float16,fp8,0,0.1448746621608734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,16,128,1,float16,float16,0,0.11006933450698853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,8,128,1,float16,float16,0,0.14173332850138345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,1,128,1,float16,float16,0,0.04044266790151596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,8,128,1,float16,fp8,0,0.14640532930692038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,1,128,1,float16,fp8,0,0.0421013335386912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,2,128,1,float16,float16,0,0.041690667470296226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,2,128,1,float16,fp8,0,0.04492799937725067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,16,128,1,float16,fp8,0,0.11037333806355794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,4,128,1,float16,float16,0,0.07040533423423767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,8,128,1,float16,float16,0,0.08090666433175404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,4,128,1,float16,fp8,0,0.07562133173147838
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,16,128,1,float16,float16,0,0.061306665341059365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,8,128,1,float16,fp8,0,0.0802346666653951
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,16,128,1,float16,fp8,0,0.05730666716893514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,1,128,1,float16,float16,0,0.022778667509555817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,1,128,1,float16,fp8,0,0.026314665873845417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,2,128,1,float16,float16,0,0.025034666061401367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,2,128,1,float16,fp8,0,0.02849599967400233
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,4,128,1,float16,float16,0,0.03048533449570338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,4,128,1,float16,fp8,0,0.03688533355792364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,8,128,1,float16,float16,0,0.037392000357309975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,8,128,1,float16,fp8,0,0.035936000446478523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,16,128,1,float16,float16,0,0.029696000119050343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,16,128,1,float16,fp8,0,0.029445332785447437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,1,128,1,float16,float16,0,0.016234666109085083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,1,128,1,float16,fp8,0,0.0184906671444575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,2,128,1,float16,float16,0,0.016943999876578648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,2,128,1,float16,fp8,0,0.01960533360640208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,4,128,1,float16,float16,0,0.019472000499566395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,4,128,1,float16,fp8,0,0.022954667607943218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,8,128,1,float16,float16,0,0.021114667256673176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,8,128,1,float16,fp8,0,0.022634667654832203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,16,128,1,float16,float16,0,0.01893866683046023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,16,128,1,float16,fp8,0,0.019274666905403137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,1,128,1,float16,float16,0,0.01523200049996376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,1,128,1,float16,fp8,0,0.01791999985774358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,8,128,1,float16,float16,0,0.015103999525308609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,8,128,1,float16,fp8,0,0.015770666301250458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,2,128,1,float16,float16,0,0.015216000378131866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,2,128,1,float16,fp8,0,0.017514667163292568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,4,128,1,float16,float16,0,0.017317333569129307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,1,128,1,float16,fp8,0,0.017770666629076004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,4,128,1,float16,fp8,0,0.019653332730134327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,16,128,1,float16,float16,0,0.013845333208640417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,16,128,1,float16,fp8,0,0.014362666755914688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,1,128,1,float16,float16,0,0.015173333386580149
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,2,128,1,float16,float16,0,0.01482133318980535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,2,128,1,float16,fp8,0,0.01735466718673706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,4,128,1,float16,float16,0,0.01524266724785169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,16,128,1,float16,fp8,0,0.013818666338920593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,4,128,1,float16,fp8,0,0.017952000101407368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,8,128,1,float16,float16,0,0.013658666362365087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,2,128,1,float16,fp8,0,0.016127999871969223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,8,128,1,float16,fp8,0,0.01379199946920077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,16,128,1,float16,float16,0,0.013568000247081121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,1,128,1,float16,float16,0,0.014261333892742792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,1,128,1,float16,fp8,0,0.016496000190575916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,2,128,1,float16,float16,0,0.014090667168299357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,4,128,1,float16,float16,0,0.014394666999578476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,4,128,1,float16,fp8,0,0.0164533331990242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,8,128,1,float16,float16,0,0.013221333424250284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,8,128,1,float16,fp8,0,0.013541333377361298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,16,128,1,float16,float16,0,0.013327999661366144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,16,128,1,float16,fp8,0,0.013397333522637686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,1,128,1,float16,float16,0,0.014186666657527288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,1,128,1,float16,fp8,0,0.01611199975013733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,2,128,1,float16,float16,0,0.013936000565687815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,2,128,1,float16,fp8,0,0.01590399940808614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,4,128,1,float16,float16,0,0.014298666268587112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,4,128,1,float16,fp8,0,0.0162773331006368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,8,128,1,float16,float16,0,0.01251199965675672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,8,128,1,float16,fp8,0,0.013104000439246496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,16,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,16,128,1,float16,fp8,0,0.013077333569526672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,1,128,1,float16,float16,0,0.01403733342885971
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,1,128,1,float16,fp8,0,0.015872000406185787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,2,128,1,float16,float16,0,0.01370666672786077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,8,128,1,float16,fp8,0,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,2,128,1,float16,fp8,0,0.01543466622630755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,4,128,1,float16,float16,0,0.013893333574136099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,4,128,1,float16,fp8,0,0.015935999651749928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,8,128,1,float16,float16,0,0.012170666207869848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,1,128,1,float16,float16,0,0.041434665520985924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,1,128,1,float16,fp8,0,0.04507733384768168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,2,128,1,float16,float16,0,0.041322665909926094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,2,128,1,float16,fp8,0,0.04438399771849314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,4,128,1,float16,float16,0,0.07547200222810109
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,4,128,1,float16,fp8,0,0.0721973329782486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,8,128,1,float16,float16,0,0.1034986674785614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,16,128,1,float16,float16,0,0.08678932984670003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,8,128,1,float16,fp8,0,0.10499733686447144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,16,128,1,float16,fp8,0,0.08277333279450734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,1,128,1,float16,float16,0,0.022831998765468597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,1,128,1,float16,fp8,0,0.0262719988822937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,2,128,1,float16,float16,0,0.024501333634058636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,4,128,1,float16,float16,0,0.03338133295377096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,4,128,1,float16,fp8,0,0.03573333223660787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,2,128,1,float16,fp8,0,0.0283146674434344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,8,128,1,float16,float16,0,0.051258668303489685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,8,128,1,float16,fp8,0,0.047456001242001854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,16,128,1,float16,fp8,0,0.04155199974775314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,1,128,1,float16,float16,0,0.016143999993801117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,16,128,1,float16,float16,0,0.04199466605981191
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,1,128,1,float16,fp8,0,0.018863999595244724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,2,128,1,float16,float16,0,0.01714666684468587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,2,128,1,float16,fp8,0,0.019354666272799175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,4,128,1,float16,float16,0,0.021242665747801464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,4,128,1,float16,fp8,0,0.022218666970729828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,1,128,1,float16,float16,0,0.015344000111023584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,8,128,1,float16,float16,0,0.0276853342851003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,8,128,1,float16,fp8,0,0.0288426677385966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,2,128,1,float16,float16,0,0.014933332800865173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,16,128,1,float16,float16,0,0.025333332518736523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,16,128,1,float16,fp8,0,0.02589333305756251
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,1,128,1,float16,fp8,0,0.01814933369557063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,2,128,1,float16,fp8,0,0.017445333302021027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,4,128,1,float16,float16,0,0.015040000279744467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,4,128,1,float16,fp8,0,0.01590399940808614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,8,128,1,float16,float16,0,0.01821333294113477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,8,128,1,float16,fp8,0,0.019285333653291065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,16,128,1,float16,float16,0,0.017136000096797943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,16,128,1,float16,fp8,0,0.017850667238235474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,1,128,1,float16,float16,0,0.015290666371583939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,1,128,1,float16,fp8,0,0.017749333133300144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,2,128,1,float16,float16,0,0.014837333311637243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,2,128,1,float16,fp8,0,0.01699200024207433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,4,128,1,float16,float16,0,0.01331199953953425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,4,128,1,float16,fp8,0,0.013701333353916803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,1,128,1,float16,fp8,0,0.01637866720557213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,8,128,1,float16,float16,0,0.01368533323208491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,8,128,1,float16,fp8,0,0.014085333794355392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,16,128,1,float16,float16,0,0.013295999417702356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,16,128,1,float16,fp8,0,0.013738666971524557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,1,128,1,float16,float16,0,0.014416000495354334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,2,128,1,float16,float16,0,0.014015999933083853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,2,128,1,float16,fp8,0,0.016176000237464905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,4,128,1,float16,float16,0,0.013269333789745966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,4,128,1,float16,fp8,0,0.013493333011865616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,8,128,1,float16,float16,0,0.013562666873137156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,8,128,1,float16,fp8,0,0.013839999834696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,16,128,1,float16,float16,0,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,16,128,1,float16,fp8,0,0.013776000589132309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,1,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,1,128,1,float16,fp8,0,0.016085332880417507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,2,128,1,float16,float16,0,0.0141546664138635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,2,128,1,float16,fp8,0,0.015674666812022526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,4,128,1,float16,fp8,0,0.013167999684810638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,8,128,1,float16,float16,0,0.01292266696691513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,8,128,1,float16,fp8,0,0.013605333864688873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,16,128,1,float16,float16,0,0.012719999998807907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,16,128,1,float16,fp8,0,0.013397333522637686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,1,128,1,float16,float16,0,0.014015999933083853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,1,128,1,float16,fp8,0,0.015919999529918034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,2,128,1,float16,float16,0,0.013616000612576803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,2,128,1,float16,fp8,0,0.015706667055686314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,4,128,1,float16,float16,0,0.012191999703645706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,4,128,1,float16,fp8,0,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,8,128,1,float16,float16,0,0.01239466667175293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,8,128,1,float16,fp8,0,0.012954667210578918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,16,128,1,float16,float16,0,0.012661332885424295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,16,128,1,float16,fp8,0,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,1,128,1,float16,float16,0,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,1,128,1,float16,fp8,0,0.01613866661985715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,2,128,1,float16,float16,0,0.013269333789745966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,2,128,1,float16,fp8,0,0.015450666348139444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,4,128,1,float16,float16,0,0.011834666132926941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,4,128,1,float16,fp8,0,0.012533333152532578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,8,128,1,float16,float16,0,0.012256000190973282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,8,128,1,float16,fp8,0,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,12,128,1,float16,float16,0,5.906272252400716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,12,1,128,1,float16,fp8,0,10.419957478841146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,12,1,128,1,float16,float16,0,11.362635294596354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,12,2,128,1,float16,float16,0,11.582171122233072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,12,2,128,1,float16,fp8,0,10.718575795491537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,12,4,128,1,float16,fp8,0,11.36572265625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,12,4,128,1,float16,float16,0,12.192559560139975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,12,128,1,float16,fp8,0,5.860005060831706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,1,128,1,float16,fp8,0,5.242074648539226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,1,128,1,float16,float16,0,5.619253158569336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,2,128,1,float16,float16,0,6.073114395141602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,12,128,1,float16,float16,0,3.044677416483561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,2,128,1,float16,fp8,0,5.359231948852539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,4,128,1,float16,float16,0,6.23634656270345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,12,128,1,float16,fp8,0,2.9637972513834634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,4,128,1,float16,fp8,0,5.708784103393555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,1,128,1,float16,float16,0,2.7175254821777344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,1,128,1,float16,fp8,0,2.943450609842936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,2,128,1,float16,float16,0,2.875493367513021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,12,128,1,float16,float16,0,1.540437380472819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,4,128,1,float16,float16,0,2.931093215942383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,2,128,1,float16,fp8,0,2.917077382405599
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,12,128,1,float16,fp8,0,1.556282679239909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,1,128,1,float16,float16,0,1.404165267944336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,4,128,1,float16,fp8,0,3.2023146947224936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,1,128,1,float16,fp8,0,1.4072906176249187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,2,128,1,float16,float16,0,1.4361066818237305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,2,128,1,float16,fp8,0,1.4148319562276204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,4,128,1,float16,float16,0,1.5642399787902832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,4,128,1,float16,fp8,0,1.585770606994629
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,12,128,1,float16,float16,0,3.6028000513712564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,12,128,1,float16,fp8,0,3.479248046875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,12,1,128,1,float16,float16,0,6.798773447672526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,12,1,128,1,float16,fp8,0,6.088229497273763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,12,2,128,1,float16,float16,0,6.779002507527669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,12,2,128,1,float16,fp8,0,6.090517044067383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,12,4,128,1,float16,float16,0,7.145322799682617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,12,4,128,1,float16,fp8,0,6.875301361083984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,12,128,1,float16,float16,0,1.901802698771159
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,12,128,1,float16,fp8,0,1.8219733238220215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,1,128,1,float16,float16,0,3.1016213099161782
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,2,128,1,float16,fp8,0,3.1131467819213867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,1,128,1,float16,fp8,0,3.0499146779378257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,2,128,1,float16,float16,0,3.190095901489258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,4,128,1,float16,float16,0,3.4838132858276367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,4,128,1,float16,fp8,0,3.400853474934896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,1,128,1,float16,float16,0,1.55731201171875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,1,128,1,float16,fp8,0,1.5728480021158855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,2,128,1,float16,float16,0,1.701807975769043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,2,128,1,float16,fp8,0,1.6110560099283855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,4,128,1,float16,float16,0,1.720234711964925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,12,128,1,float16,float16,0,0.9767093658447266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,4,128,1,float16,fp8,0,1.8407360712687175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,12,128,1,float16,fp8,0,0.9567200342814127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,1,128,1,float16,float16,0,0.8182506561279297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,1,128,1,float16,fp8,0,0.8342026869455973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,2,128,1,float16,float16,0,0.886245330174764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,2,128,1,float16,fp8,0,0.857754627863566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,4,128,1,float16,float16,0,0.8994506994883219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,4,128,1,float16,fp8,0,0.9534239768981934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,12,1,128,1,float16,float16,0,4.469936052958171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,12,1,128,1,float16,fp8,0,4.229056040445964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,12,128,1,float16,float16,0,2.522496064503988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,12,2,128,1,float16,float16,0,4.437669436136882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,12,128,1,float16,fp8,0,2.5911946296691895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,12,2,128,1,float16,fp8,0,4.351829210917155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,12,4,128,1,float16,float16,0,4.936362584431966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,12,4,128,1,float16,fp8,0,4.856682777404785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,1,128,1,float16,fp8,0,2.1585493087768555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,1,128,1,float16,float16,0,2.436768054962158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,2,128,1,float16,float16,0,2.2498879432678223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,12,128,1,float16,float16,0,1.331760009129842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,2,128,1,float16,fp8,0,2.4616212844848633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,12,128,1,float16,fp8,0,1.3256693681081135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,4,128,1,float16,float16,0,2.50492795308431
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,4,128,1,float16,fp8,0,2.470656077067057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,1,128,1,float16,float16,0,1.1106080214182537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,1,128,1,float16,fp8,0,1.2428053220113118
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,2,128,1,float16,fp8,0,1.2353706359863281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,2,128,1,float16,float16,0,1.179093360900879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,4,128,1,float16,float16,0,1.251802682876587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,12,128,1,float16,float16,0,0.7135626475016276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,4,128,1,float16,fp8,0,1.28001602490743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,12,128,1,float16,fp8,0,0.7090720335642496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,1,128,1,float16,float16,0,0.5930346647898356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,2,128,1,float16,float16,0,0.6224639813105265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,1,128,1,float16,fp8,0,0.6428373257319132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,2,128,1,float16,fp8,0,0.6540799935658773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,4,128,1,float16,float16,0,0.661359985669454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,4,128,1,float16,fp8,0,0.7015893459320068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,12,1,128,1,float16,float16,0,6.113781611124675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,12,128,1,float16,float16,0,3.3461974461873374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,12,1,128,1,float16,fp8,0,5.486288070678711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,12,128,1,float16,fp8,0,3.458906809488932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,12,2,128,1,float16,float16,0,6.236650466918945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,12,2,128,1,float16,fp8,0,5.805168151855469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,12,4,128,1,float16,float16,0,6.653514862060547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,12,4,128,1,float16,fp8,0,6.65451176961263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,1,128,1,float16,float16,0,2.9347092310587564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,1,128,1,float16,fp8,0,2.7856372197469077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,2,128,1,float16,float16,0,3.0008214314778647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,2,128,1,float16,fp8,0,3.101658821105957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,12,128,1,float16,float16,0,1.7093812624613445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,4,128,1,float16,float16,0,3.2434558868408203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,12,128,1,float16,fp8,0,1.8582132657368977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,4,128,1,float16,fp8,0,3.5555038452148438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,1,128,1,float16,float16,0,1.4055412610371907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,1,128,1,float16,fp8,0,1.5742400487263997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,2,128,1,float16,float16,0,1.5198133786519368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,2,128,1,float16,fp8,0,1.5193012555440266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,12,128,1,float16,float16,0,0.9009866714477539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,4,128,1,float16,float16,0,1.6658825874328613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,12,128,1,float16,fp8,0,0.9040640195210775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,1,128,1,float16,float16,0,0.7544480164845785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,4,128,1,float16,fp8,0,1.7464906374613445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,1,128,1,float16,fp8,0,0.7678026358286539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,2,128,1,float16,float16,0,0.775493303934733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,2,128,1,float16,fp8,0,0.8159146308898926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,4,128,1,float16,float16,0,0.8674399852752686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,12,128,1,float16,float16,0,0.4899839957555135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,4,128,1,float16,fp8,0,0.880021333694458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,12,128,1,float16,fp8,0,0.5173279841740926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,1,128,1,float16,float16,0,0.39646931489308673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,1,128,1,float16,fp8,0,0.42344534397125244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,2,128,1,float16,float16,0,0.42124799887339276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,2,128,1,float16,fp8,0,0.4541920026143392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,4,128,1,float16,float16,0,0.46666133403778076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,4,128,1,float16,fp8,0,0.47968534628550213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,12,1,128,1,float16,float16,0,3.248602549235026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,12,1,128,1,float16,fp8,0,3.331109364827474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,12,128,1,float16,float16,0,2.0997279485066733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,12,2,128,1,float16,float16,0,3.4188639322916665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,12,128,1,float16,fp8,0,2.1365440686543784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,12,2,128,1,float16,fp8,0,3.4384533564249673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,12,4,128,1,float16,float16,0,3.915679931640625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,1,128,1,float16,float16,0,1.6747786204020183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,1,128,1,float16,fp8,0,1.6997013092041016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,2,128,1,float16,float16,0,1.7676374117533367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,2,128,1,float16,fp8,0,1.8859680493672688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,4,128,1,float16,float16,0,1.952624003092448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,12,128,1,float16,float16,0,1.1181706587473552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,12,128,1,float16,fp8,0,1.178810675938924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,1,128,1,float16,float16,0,0.839792013168335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,4,128,1,float16,fp8,0,2.0380214055379233
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,12,4,128,1,float16,fp8,0,4.084725379943848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,1,128,1,float16,fp8,0,0.9101440111796061
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,2,128,1,float16,float16,0,0.8974560101826986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,2,128,1,float16,fp8,0,0.9337226549784342
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,4,128,1,float16,float16,0,1.0176160335540771
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,12,128,1,float16,float16,0,0.5556053320566813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,4,128,1,float16,fp8,0,1.083674669265747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,12,128,1,float16,fp8,0,0.5829279820124308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,1,128,1,float16,float16,0,0.43813331921895343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,1,128,1,float16,fp8,0,0.4747519890467326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,12,128,1,float16,float16,0,0.3129439949989319
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,2,128,1,float16,float16,0,0.4734400113423665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,2,128,1,float16,fp8,0,0.4914986689885457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,4,128,1,float16,float16,0,0.5333760182062784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,4,128,1,float16,fp8,0,0.5625600020090739
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,12,128,1,float16,fp8,0,0.32790933052698773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,1,128,1,float16,float16,0,0.24573334058125815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,1,128,1,float16,fp8,0,0.2632373372713725
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,2,128,1,float16,float16,0,0.2659306724866231
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,2,128,1,float16,fp8,0,0.2730506658554077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,4,128,1,float16,float16,0,0.29501867294311523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,4,128,1,float16,fp8,0,0.3182186683019002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,12,1,128,1,float16,float16,0,3.05727481842041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,12,1,128,1,float16,fp8,0,3.109034538269043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,12,2,128,1,float16,float16,0,3.3207839330037436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,12,128,1,float16,float16,0,2.0795839627583823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,12,2,128,1,float16,fp8,0,3.336698532104492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,12,128,1,float16,fp8,0,2.2062400182088218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,12,4,128,1,float16,float16,0,3.9287894566853843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,1,128,1,float16,float16,0,1.6075040499369304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,12,4,128,1,float16,fp8,0,4.174655914306641
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,1,128,1,float16,fp8,0,1.5793226559956868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,2,128,1,float16,float16,0,1.6825706164042156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,2,128,1,float16,fp8,0,1.7055999437967937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,12,128,1,float16,float16,0,1.081653356552124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,4,128,1,float16,float16,0,1.9945707321166992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,4,128,1,float16,fp8,0,2.1226560274759927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,1,128,1,float16,float16,0,0.783082644144694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,12,128,1,float16,fp8,0,1.1910719871520996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,1,128,1,float16,fp8,0,0.8115893205006918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,2,128,1,float16,float16,0,0.8743147055308024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,2,128,1,float16,fp8,0,0.8803359667460123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,4,128,1,float16,float16,0,1.0135573546091716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,12,128,1,float16,float16,0,0.5573866764704386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,4,128,1,float16,fp8,0,1.0711092948913574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,1,128,1,float16,float16,0,0.4142560164133708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,12,128,1,float16,fp8,0,0.5990186532338461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,1,128,1,float16,fp8,0,0.42955199877421063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,2,128,1,float16,float16,0,0.463210662206014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,2,128,1,float16,fp8,0,0.4691733519236247
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,4,128,1,float16,float16,0,0.5261439879735311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,4,128,1,float16,fp8,0,0.5734399954477946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,12,128,1,float16,float16,0,0.3033439914385478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,12,128,1,float16,fp8,0,0.3285973270734151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,1,128,1,float16,float16,0,0.2286506692568461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,1,128,1,float16,fp8,0,0.2450986703236898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,2,128,1,float16,float16,0,0.24929600954055786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,2,128,1,float16,fp8,0,0.25964800516764325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,4,128,1,float16,float16,0,0.2884693344434102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,4,128,1,float16,fp8,0,0.3137813409169515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,12,128,1,float16,float16,0,0.17484800020853677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,12,128,1,float16,fp8,0,0.19051200151443481
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,1,128,1,float16,float16,0,0.12800000111262003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,1,128,1,float16,fp8,0,0.13426132996877035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,2,128,1,float16,float16,0,0.1407360037167867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,2,128,1,float16,fp8,0,0.14840533336003622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,4,128,1,float16,float16,0,0.16780267159144083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,4,128,1,float16,fp8,0,0.17299733559290567
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,12,1,128,1,float16,float16,0,1.8127733866373699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,12,1,128,1,float16,fp8,0,1.909813404083252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,12,2,128,1,float16,float16,0,1.9504586855570476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,12,2,128,1,float16,fp8,0,2.043194611867269
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,12,128,1,float16,float16,0,1.3475573857625325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,12,4,128,1,float16,float16,0,2.4756800333658853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,12,4,128,1,float16,fp8,0,2.6657546361287436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,1,128,1,float16,float16,0,0.9224960009256998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,2,128,1,float16,float16,0,1.0236639976501465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,12,128,1,float16,fp8,0,1.4075039227803547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,1,128,1,float16,fp8,0,0.9789706865946451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,2,128,1,float16,fp8,0,1.0511679649353027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,4,128,1,float16,float16,0,1.251482645670573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,12,128,1,float16,float16,0,0.6857919692993164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,12,128,1,float16,fp8,0,0.7324639956156412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,4,128,1,float16,fp8,0,1.3575572967529297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,1,128,1,float16,float16,0,0.4752693176269531
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,1,128,1,float16,fp8,0,0.5164479811986288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,2,128,1,float16,float16,0,0.5423146486282349
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,2,128,1,float16,fp8,0,0.547823985417684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,4,128,1,float16,float16,0,0.6408533255259196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,12,128,1,float16,float16,0,0.36517333984375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,4,128,1,float16,fp8,0,0.7011573314666748
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,12,128,1,float16,fp8,0,0.3950186570485433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,1,128,1,float16,float16,0,0.25597866376241046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,1,128,1,float16,fp8,0,0.28055999676386517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,2,128,1,float16,float16,0,0.2916426658630371
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,2,128,1,float16,fp8,0,0.2990986704826355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,4,128,1,float16,float16,0,0.34111467997233075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,4,128,1,float16,fp8,0,0.3803253173828125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,12,128,1,float16,float16,0,0.20562666654586792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,12,128,1,float16,fp8,0,0.2209013303120931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,1,128,1,float16,float16,0,0.14728533228238425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,1,128,1,float16,fp8,0,0.15729066729545593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,2,128,1,float16,float16,0,0.16396266222000122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,4,128,1,float16,float16,0,0.190175990263621
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,2,128,1,float16,fp8,0,0.16821332772572836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,4,128,1,float16,fp8,0,0.21076265970865884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,12,128,1,float16,float16,0,0.11917333801587422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,12,128,1,float16,fp8,0,0.13310399651527405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,1,128,1,float16,float16,0,0.0860640009244283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,1,128,1,float16,fp8,0,0.09539733330408733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,2,128,1,float16,float16,0,0.09071999788284302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,2,128,1,float16,fp8,0,0.09794132908185323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,4,128,1,float16,fp8,0,0.11588266491889954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,4,128,1,float16,float16,0,0.10470400253931682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,12,1,128,1,float16,float16,0,1.787552038828532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,12,1,128,1,float16,fp8,0,1.9122133255004883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,12,2,128,1,float16,float16,0,2.0757600466410318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,12,128,1,float16,float16,0,1.4661760330200195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,12,2,128,1,float16,fp8,0,2.0803306897481284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,12,128,1,float16,fp8,0,1.5384373664855957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,12,4,128,1,float16,float16,0,2.638357321421305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,1,128,1,float16,float16,0,0.9144586722056071
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,12,4,128,1,float16,fp8,0,2.874426523844401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,1,128,1,float16,fp8,0,0.9751893679300944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,2,128,1,float16,float16,0,1.0588106314341228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,2,128,1,float16,fp8,0,1.0618666807810466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,12,128,1,float16,float16,0,0.7413012981414795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,4,128,1,float16,float16,0,1.3399146397908528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,1,128,1,float16,float16,0,0.47488001982371014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,4,128,1,float16,fp8,0,1.484373410542806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,12,128,1,float16,fp8,0,0.8012639681498209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,1,128,1,float16,fp8,0,0.5064906676610311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,2,128,1,float16,float16,0,0.5449706713358561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,2,128,1,float16,fp8,0,0.5514506498972574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,4,128,1,float16,float16,0,0.6892746289571127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,4,128,1,float16,fp8,0,0.7555519739786783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,12,128,1,float16,float16,0,0.3909440040588379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,12,128,1,float16,fp8,0,0.41921599706013996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,1,128,1,float16,float16,0,0.254095991452535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,1,128,1,float16,fp8,0,0.2691306670506795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,2,128,1,float16,float16,0,0.28893866141637164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,2,128,1,float16,fp8,0,0.2940160036087036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,4,128,1,float16,float16,0,0.361135999361674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,4,128,1,float16,fp8,0,0.39768000443776447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,12,128,1,float16,float16,0,0.21221866210301718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,12,128,1,float16,fp8,0,0.22822932402292886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,1,128,1,float16,float16,0,0.14172800381978354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,1,128,1,float16,fp8,0,0.1514346698919932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,2,128,1,float16,float16,0,0.1618880033493042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,2,128,1,float16,fp8,0,0.166293332974116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,4,128,1,float16,float16,0,0.1959679921468099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,4,128,1,float16,fp8,0,0.21820799509684244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,12,128,1,float16,float16,0,0.12008532881736755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,12,128,1,float16,fp8,0,0.1329759955406189
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,1,128,1,float16,float16,0,0.07982933521270752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,1,128,1,float16,fp8,0,0.08608532945315044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,2,128,1,float16,float16,0,0.08727999528249104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,2,128,1,float16,fp8,0,0.09292266766230266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,4,128,1,float16,float16,0,0.1114453375339508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,4,128,1,float16,fp8,0,0.11452266573905945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,12,128,1,float16,float16,0,0.07294933497905731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,12,128,1,float16,fp8,0,0.07612800101439159
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,1,128,1,float16,float16,0,0.05133333305517832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,1,128,1,float16,fp8,0,0.05672533313433329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,2,128,1,float16,float16,0,0.0537013312180837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,2,128,1,float16,fp8,0,0.060991997520128884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,4,128,1,float16,float16,0,0.06155733267466227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,4,128,1,float16,fp8,0,0.07051733136177063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,12,1,128,1,float16,float16,0,1.1218132972717285
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,12,1,128,1,float16,fp8,0,1.2220853169759114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,12,2,128,1,float16,float16,0,1.2799306710561116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,12,128,1,float16,float16,0,0.9794613520304362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,12,2,128,1,float16,fp8,0,1.346709410349528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,12,4,128,1,float16,float16,0,1.7610026995340984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,12,4,128,1,float16,fp8,0,1.9776852925618489
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,1,128,1,float16,float16,0,0.576906681060791
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,12,128,1,float16,fp8,0,1.0729440053304036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,1,128,1,float16,fp8,0,0.6287093162536621
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,2,128,1,float16,float16,0,0.6800320148468018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,2,128,1,float16,fp8,0,0.692026694615682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,4,128,1,float16,float16,0,0.8920426368713379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,12,128,1,float16,float16,0,0.5030879974365234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,4,128,1,float16,fp8,0,1.005669355392456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,1,128,1,float16,float16,0,0.3055573304494222
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,12,128,1,float16,fp8,0,0.5466560125350952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,1,128,1,float16,fp8,0,0.32733333110809326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,2,128,1,float16,float16,0,0.36213334401448566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,2,128,1,float16,fp8,0,0.36552000045776367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,4,128,1,float16,float16,0,0.4708160161972046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,4,128,1,float16,fp8,0,0.5235786835352579
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,12,128,1,float16,float16,0,0.2688159942626953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,12,128,1,float16,fp8,0,0.2948213418324788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,2,128,1,float16,float16,0,0.19689599672953287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,1,128,1,float16,float16,0,0.16538133223851523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,1,128,1,float16,fp8,0,0.18057066202163696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,2,128,1,float16,fp8,0,0.1974666714668274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,4,128,1,float16,float16,0,0.2455199956893921
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,4,128,1,float16,fp8,0,0.2801813284556071
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,12,128,1,float16,float16,0,0.14568533500035605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,2,128,1,float16,float16,0,0.10744532942771912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,12,128,1,float16,fp8,0,0.1660266617933909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,1,128,1,float16,float16,0,0.09511466821034749
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,1,128,1,float16,fp8,0,0.10064533352851868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,2,128,1,float16,fp8,0,0.11388799548149109
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,4,128,1,float16,float16,0,0.13433067003885904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,4,128,1,float16,fp8,0,0.15412799517313638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,12,128,1,float16,float16,0,0.08426133791605632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,12,128,1,float16,fp8,0,0.09578133622805278
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,1,128,1,float16,float16,0,0.054330666859944664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,1,128,1,float16,fp8,0,0.06185600161552429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,2,128,1,float16,float16,0,0.05938133100668589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,2,128,1,float16,fp8,0,0.06534400085608165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,4,128,1,float16,float16,0,0.07352533439795177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,4,128,1,float16,fp8,0,0.08284799754619598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,12,128,1,float16,float16,0,0.04712533454100291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,12,128,1,float16,fp8,0,0.05459733307361603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,1,128,1,float16,float16,0,0.038575999438762665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,1,128,1,float16,fp8,0,0.04391466577847799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,2,128,1,float16,float16,0,0.03993066648642222
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,2,128,1,float16,fp8,0,0.04619200030962626
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,4,128,1,float16,float16,0,0.043653334180514015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,4,128,1,float16,fp8,0,0.051776001850763954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,12,1,128,1,float16,float16,0,1.3604639371236165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,12,1,128,1,float16,fp8,0,1.4470346768697102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,12,128,1,float16,fp8,0,1.335466702779134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,12,128,1,float16,float16,0,1.232485294342041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,12,2,128,1,float16,float16,0,1.6385226249694824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,12,2,128,1,float16,fp8,0,1.6065279642740886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,1,128,1,float16,float16,0,0.695354700088501
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,1,128,1,float16,fp8,0,0.740389347076416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,2,128,1,float16,float16,0,0.8329386711120605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,12,4,128,1,float16,float16,0,2.1812159220377603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,12,4,128,1,float16,fp8,0,2.453359921773275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,4,128,1,float16,float16,0,1.1216586430867512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,2,128,1,float16,fp8,0,0.826794703801473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,12,128,1,float16,float16,0,0.6256213188171387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,12,128,1,float16,fp8,0,0.6885546843210856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,4,128,1,float16,fp8,0,1.2516106764475505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,1,128,1,float16,float16,0,0.3615413506825765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,12,128,1,float16,float16,0,0.3272479971249898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,4,128,1,float16,float16,0,0.5713866551717123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,1,128,1,float16,float16,0,0.19339199860890707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,2,128,1,float16,fp8,0,0.42530667781829834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,4,128,1,float16,fp8,0,0.6351413329442342
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,1,128,1,float16,fp8,0,0.20468266805013022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,2,128,1,float16,fp8,0,0.226474662621816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,12,128,1,float16,fp8,0,0.35757867495218915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,2,128,1,float16,float16,0,0.2342026631037394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,4,128,1,float16,float16,0,0.3003200093905131
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,12,128,1,float16,float16,0,0.1752906640370687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,4,128,1,float16,fp8,0,0.33397332827250165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,12,128,1,float16,fp8,0,0.19339734315872192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,1,128,1,float16,float16,0,0.10750400026639302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,2,128,1,float16,float16,0,0.429909348487854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,1,128,1,float16,fp8,0,0.38278400897979736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,2,128,1,float16,fp8,0,0.13004266222318014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,1,128,1,float16,fp8,0,0.11546666423479716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,2,128,1,float16,float16,0,0.1272266705830892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,4,128,1,float16,float16,0,0.1596213380495707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,4,128,1,float16,fp8,0,0.1802026629447937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,12,128,1,float16,float16,0,0.09744532903035481
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,1,128,1,float16,fp8,0,0.06362666686375935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,1,128,1,float16,float16,0,0.05898666878541311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,2,128,1,float16,float16,0,0.06890133519967397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,12,128,1,float16,fp8,0,0.1104693313439687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,4,128,1,float16,fp8,0,0.09430399537086487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,2,128,1,float16,fp8,0,0.06859200199445088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,4,128,1,float16,float16,0,0.08729066451390584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,12,128,1,float16,float16,0,0.056218668818473816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,12,128,1,float16,fp8,0,0.058864002426465355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,1,128,1,float16,float16,0,0.03760000069936117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,1,128,1,float16,fp8,0,0.04068800061941147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,4,128,1,float16,fp8,0,0.052671998739242554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,2,128,1,float16,float16,0,0.04119999955097834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,2,128,1,float16,fp8,0,0.04499199986457825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,4,128,1,float16,float16,0,0.046896000703175865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,12,128,1,float16,float16,0,0.031231999397277832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,12,128,1,float16,fp8,0,0.03572266548871994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,1,128,1,float16,float16,0,0.02638400097688039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,1,128,1,float16,fp8,0,0.028890666862328846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,2,128,1,float16,float16,0,0.027669332921504974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,2,128,1,float16,fp8,0,0.030581332743167877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,4,128,1,float16,float16,0,0.030608000854651134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,4,128,1,float16,fp8,0,0.03519999980926514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,12,1,128,1,float16,float16,0,0.9100053310394287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,12,128,1,float16,float16,0,0.9998186429341634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,12,128,1,float16,fp8,0,1.1250826517740886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,12,1,128,1,float16,fp8,0,1.0318346818288167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,12,2,128,1,float16,float16,0,1.174730698267619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,12,2,128,1,float16,fp8,0,1.2033653259277344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,1,128,1,float16,float16,0,0.4689706563949585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,12,4,128,1,float16,float16,0,1.729200045267741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,1,128,1,float16,fp8,0,0.5331093470255533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,2,128,1,float16,float16,0,0.599237322807312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,12,4,128,1,float16,fp8,0,2.0580053329467773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,2,128,1,float16,fp8,0,0.6196853319803873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,4,128,1,float16,float16,0,0.8827679951985677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,12,128,1,float16,float16,0,0.5106720129648844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,4,128,1,float16,fp8,0,1.0542559623718262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,1,128,1,float16,float16,0,0.24529065688451132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,1,128,1,float16,fp8,0,0.275706668694814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,12,128,1,float16,fp8,0,0.5924319823582967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,2,128,1,float16,fp8,0,0.32460800806681317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,2,128,1,float16,float16,0,0.30906667311986286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,4,128,1,float16,float16,0,0.45232534408569336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,1,128,1,float16,float16,0,0.1304266651471456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,12,128,1,float16,fp8,0,0.31092800696690875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,12,128,1,float16,float16,0,0.2654613256454468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,1,128,1,float16,fp8,0,0.15054399768511453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,4,128,1,float16,fp8,0,0.5436319907506307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,2,128,1,float16,float16,0,0.16563733418782553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,2,128,1,float16,fp8,0,0.1746506690979004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,1,128,1,float16,float16,0,0.07601066430409749
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,4,128,1,float16,float16,0,0.23581333955128989
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,4,128,1,float16,fp8,0,0.2831786672274272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,12,128,1,float16,float16,0,0.14231466253598532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,12,128,1,float16,fp8,0,0.1684053341547648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,1,128,1,float16,fp8,0,0.08708266417185466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,2,128,1,float16,float16,0,0.09481599926948547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,2,128,1,float16,fp8,0,0.10056533416112264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,4,128,1,float16,float16,0,0.12749333182970682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,4,128,1,float16,fp8,0,0.15376533071200052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,12,128,1,float16,float16,0,0.07993066807587941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,12,128,1,float16,fp8,0,0.0997279981772105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,1,128,1,float16,float16,0,0.041850666205088295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,1,128,1,float16,fp8,0,0.049626668294270836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,2,128,1,float16,float16,0,0.04887466629346212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,2,128,1,float16,fp8,0,0.05468266705671946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,1,128,1,float16,float16,0,0.028192001084486645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,4,128,1,float16,float16,0,0.07082133491834004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,4,128,1,float16,fp8,0,0.07543466488520305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,12,128,1,float16,float16,0,0.044863998889923096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,12,128,1,float16,fp8,0,0.05133866767088572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,1,128,1,float16,fp8,0,0.033973333736260734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,2,128,1,float16,float16,0,0.030597334106763203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,2,128,1,float16,fp8,0,0.03748266647259394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,4,128,1,float16,float16,0,0.035887998839219414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,4,128,1,float16,fp8,0,0.04507199923197428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,12,128,1,float16,float16,0,0.024879999458789825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,12,128,1,float16,fp8,0,0.030832000076770782
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,1,128,1,float16,float16,0,0.020058666666348774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,1,128,1,float16,fp8,0,0.023647998770078022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,2,128,1,float16,float16,0,0.021386665602525074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,2,128,1,float16,fp8,0,0.02647999922434489
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,4,128,1,float16,float16,0,0.02401600033044815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,4,128,1,float16,fp8,0,0.03012799968322118
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,12,128,1,float16,float16,0,0.02184533327817917
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,12,128,1,float16,fp8,0,0.02712533374627431
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,1,128,1,float16,float16,0,0.01953599974513054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,1,128,1,float16,fp8,0,0.022341333329677582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,2,128,1,float16,float16,0,0.019808000574509304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,2,128,1,float16,fp8,0,0.02493866781393687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,4,128,1,float16,float16,0,0.02125866711139679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,4,128,1,float16,fp8,0,0.026464000344276428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,12,1,128,1,float16,float16,0,0.39470934867858887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,12,1,128,1,float16,fp8,0,0.45316799481709796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,12,2,128,1,float16,float16,0,0.5181546608606974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,12,2,128,1,float16,fp8,0,0.5483200152715048
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,12,128,1,float16,fp8,0,0.5346346696217855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,1,128,1,float16,float16,0,0.2079413334528605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,12,128,1,float16,float16,0,0.4797866741816203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,12,4,128,1,float16,float16,0,0.8161493142445883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,12,4,128,1,float16,fp8,0,0.9753973484039307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,1,128,1,float16,fp8,0,0.23967466751734415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,2,128,1,float16,float16,0,0.26994667450586957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,4,128,1,float16,float16,0,0.4163466691970825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,2,128,1,float16,fp8,0,0.2893173297246297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,12,128,1,float16,float16,0,0.24970134099324545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,1,128,1,float16,float16,0,0.114464004834493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,1,128,1,float16,fp8,0,0.13316266735394797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,12,128,1,float16,fp8,0,0.2820906639099121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,2,128,1,float16,float16,0,0.15217600266138712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,2,128,1,float16,fp8,0,0.15795733531316122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,4,128,1,float16,fp8,0,0.4996960163116455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,12,128,1,float16,float16,0,0.1341919998327891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,12,128,1,float16,fp8,0,0.15503467122713724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,4,128,1,float16,fp8,0,0.26265066862106323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,4,128,1,float16,float16,0,0.2181333303451538
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,1,128,1,float16,fp8,0,0.07673066854476929
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,1,128,1,float16,float16,0,0.06743999818960826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,2,128,1,float16,float16,0,0.08262933293978374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,4,128,1,float16,float16,0,0.11654399832089742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,12,128,1,float16,float16,0,0.07474133372306824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,1,128,1,float16,float16,0,0.03507733345031738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,4,128,1,float16,fp8,0,0.14243200421333313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,2,128,1,float16,fp8,0,0.09054932991663615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,12,128,1,float16,fp8,0,0.08845866719881694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,1,128,1,float16,fp8,0,0.04083200047413508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,2,128,1,float16,float16,0,0.04115733255942663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,2,128,1,float16,fp8,0,0.04706666866938273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,4,128,1,float16,float16,0,0.06322666505972545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,4,128,1,float16,fp8,0,0.06717333197593689
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,12,128,1,float16,float16,0,0.038933334251244865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,12,128,1,float16,fp8,0,0.04407466451327006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,1,128,1,float16,float16,0,0.023989332218964893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,1,128,1,float16,fp8,0,0.028213332096735638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,2,128,1,float16,float16,0,0.02622933437426885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,2,128,1,float16,fp8,0,0.0323786661028862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,4,128,1,float16,float16,0,0.031343999008337654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,4,128,1,float16,fp8,0,0.039146666725476585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,12,128,1,float16,float16,0,0.021840001145998638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,12,128,1,float16,fp8,0,0.027450665831565857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,1,128,1,float16,float16,0,0.01770666614174843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,1,128,1,float16,fp8,0,0.02011200040578842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,2,128,1,float16,float16,0,0.018464000274737675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,2,128,1,float16,fp8,0,0.022661333282788593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,4,128,1,float16,float16,0,0.02109333376089732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,4,128,1,float16,fp8,0,0.026309333741664886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,12,128,1,float16,float16,0,0.01887999971707662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,12,128,1,float16,fp8,0,0.023914667467276256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,1,128,1,float16,float16,0,0.016762666404247284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,1,128,1,float16,fp8,0,0.01945066700379054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,2,128,1,float16,float16,0,0.017184000462293625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,12,128,1,float16,fp8,0,0.02201066662867864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,2,128,1,float16,fp8,0,0.02117866774400075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,4,128,1,float16,float16,0,0.01844266677896182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,4,128,1,float16,fp8,0,0.02266666789849599
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,12,128,1,float16,float16,0,0.01852799952030182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,1,128,1,float16,float16,0,0.016549333930015564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,1,128,1,float16,fp8,0,0.01931200052301089
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,2,128,1,float16,float16,0,0.016522667060295742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,2,128,1,float16,fp8,0,0.02027733375628789
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,4,128,1,float16,float16,0,0.01684800038735072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,4,128,1,float16,fp8,0,0.02072000006834666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,12,1,128,1,float16,float16,0,0.20124266544977823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,12,1,128,1,float16,fp8,0,0.21973333756128946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,12,2,128,1,float16,float16,0,0.27180800835291546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,12,2,128,1,float16,fp8,0,0.2606826623280843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,12,128,1,float16,float16,0,0.24285866816838583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,1,128,1,float16,float16,0,0.11017066240310669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,12,128,1,float16,fp8,0,0.2602399984995524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,12,4,128,1,float16,float16,0,0.40307732423146564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,1,128,1,float16,fp8,0,0.12339199582735698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,12,4,128,1,float16,fp8,0,0.47542933622996014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,2,128,1,float16,float16,0,0.14505599935849509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,2,128,1,float16,fp8,0,0.14406399925549826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,4,128,1,float16,fp8,0,0.24862400690714517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,1,128,1,float16,float16,0,0.06468800206979115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,12,128,1,float16,float16,0,0.13037332892417908
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,4,128,1,float16,float16,0,0.21146667003631592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,2,128,1,float16,float16,0,0.0813973347345988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,12,128,1,float16,fp8,0,0.14291200041770935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,2,128,1,float16,fp8,0,0.08306666711966197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,1,128,1,float16,fp8,0,0.07107200225194295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,4,128,1,float16,float16,0,0.11388799548149109
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,12,128,1,float16,float16,0,0.07179733117421468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,4,128,1,float16,fp8,0,0.1325706640879313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,12,128,1,float16,fp8,0,0.07909866670767467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,1,128,1,float16,float16,0,0.031136001149813335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,1,128,1,float16,fp8,0,0.03598399957021078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,12,128,1,float16,float16,0,0.038480001191298165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,2,128,1,float16,float16,0,0.03844266633192698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,2,128,1,float16,fp8,0,0.040565334260463715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,4,128,1,float16,float16,0,0.061066667238871254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,4,128,1,float16,fp8,0,0.05879466732343038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,12,128,1,float16,fp8,0,0.038165333370367684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,1,128,1,float16,float16,0,0.02218666672706604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,1,128,1,float16,fp8,0,0.026186667382717133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,2,128,1,float16,float16,0,0.02531733363866806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,12,128,1,float16,float16,0,0.02060266708334287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,12,128,1,float16,fp8,0,0.023354666928450268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,2,128,1,float16,fp8,0,0.028762665887673695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,4,128,1,float16,float16,0,0.030426666140556335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,4,128,1,float16,fp8,0,0.035274667044480644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,1,128,1,float16,float16,0,0.016197333733240765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,1,128,1,float16,fp8,0,0.01882133384545644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,2,128,1,float16,fp8,0,0.01974933346112569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,2,128,1,float16,float16,0,0.017301333447297413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,4,128,1,float16,float16,0,0.01964266722400983
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,4,128,1,float16,fp8,0,0.02319466571013133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,12,128,1,float16,float16,0,0.017103999853134155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,12,128,1,float16,fp8,0,0.0198186660806338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,1,128,1,float16,float16,0,0.015311999867359797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,1,128,1,float16,fp8,0,0.0180479995906353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,2,128,1,float16,float16,0,0.01543466622630755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,2,128,1,float16,fp8,0,0.017605333278576534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,4,128,1,float16,float16,0,0.01717866708834966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,4,128,1,float16,fp8,0,0.019637333850065868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,12,128,1,float16,float16,0,0.015418666104475657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,12,128,1,float16,fp8,0,0.017674667139848072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,4,128,1,float16,float16,0,0.015605332950750986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,1,128,1,float16,float16,0,0.015072000523408255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,1,128,1,float16,fp8,0,0.01749333366751671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,2,128,1,float16,float16,0,0.015125333021084467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,2,128,1,float16,fp8,0,0.01743999992807706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,4,128,1,float16,fp8,0,0.017498667041460674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,12,128,1,float16,float16,0,0.014416000495354334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,12,128,1,float16,fp8,0,0.016341333587964375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,1,128,1,float16,float16,0,0.014080000420411428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,4,128,1,float16,fp8,0,0.016389333953460056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,1,128,1,float16,fp8,0,0.0161920003592968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,2,128,1,float16,float16,0,0.014303999642531076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,2,128,1,float16,fp8,0,0.016303999970356624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,12,1,128,1,float16,float16,0,0.10809600353240967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,12,1,128,1,float16,fp8,0,0.12370666861534119
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,12,2,128,1,float16,fp8,0,0.1442453364531199
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,12,2,128,1,float16,float16,0,0.14323733250300089
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,12,128,1,float16,float16,0,0.137061337629954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,12,128,1,float16,fp8,0,0.14085333546002707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,12,4,128,1,float16,float16,0,0.22686400016148886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,12,4,128,1,float16,fp8,0,0.2679413358370463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,1,128,1,float16,float16,0,0.063701331615448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,1,128,1,float16,fp8,0,0.07196799914042155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,2,128,1,float16,float16,0,0.08239466448624928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,2,128,1,float16,fp8,0,0.08227199812730153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,4,128,1,float16,float16,0,0.1270133356253306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,12,128,1,float16,float16,0,0.07626666625340779
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,1,128,1,float16,fp8,0,0.03583999971548716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,4,128,1,float16,fp8,0,0.14214932918548584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,12,128,1,float16,fp8,0,0.07601599891980489
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,1,128,1,float16,float16,0,0.030896000564098358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,2,128,1,float16,float16,0,0.03824000060558319
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,2,128,1,float16,fp8,0,0.039962666730086006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,4,128,1,float16,float16,0,0.06518400212128957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,12,128,1,float16,float16,0,0.038058665891488395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,4,128,1,float16,fp8,0,0.06746666630109151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,12,128,1,float16,fp8,0,0.03443733354409536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,1,128,1,float16,float16,0,0.02233600119749705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,1,128,1,float16,fp8,0,0.025839999318122864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,2,128,1,float16,float16,0,0.024885334074497223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,2,128,1,float16,fp8,0,0.028629332780838013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,4,128,1,float16,float16,0,0.029765332738558452
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,4,128,1,float16,fp8,0,0.035455999275048576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,12,128,1,float16,float16,0,0.021712000171343487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,12,128,1,float16,fp8,0,0.022874665757020313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,1,128,1,float16,float16,0,0.0161920003592968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,1,128,1,float16,fp8,0,0.018405333161354065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,2,128,1,float16,float16,0,0.016895999511082966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,2,128,1,float16,fp8,0,0.02012266715367635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,4,128,1,float16,float16,0,0.019589333484570186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,4,128,1,float16,fp8,0,0.022842665513356526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,12,128,1,float16,float16,0,0.01545599972208341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,12,128,1,float16,fp8,0,0.01637866720557213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,1,128,1,float16,float16,0,0.015200000256299973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,1,128,1,float16,fp8,0,0.01786133274435997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,2,128,1,float16,float16,0,0.015605332950750986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,2,128,1,float16,fp8,0,0.018021332720915478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,4,128,1,float16,float16,0,0.016682667036851246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,4,128,1,float16,fp8,0,0.019610666980346043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,12,128,1,float16,float16,0,0.013584000368913015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,12,128,1,float16,fp8,0,0.014096000542243322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,1,128,1,float16,float16,0,0.015087999403476715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,1,128,1,float16,fp8,0,0.017498667041460674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,2,128,1,float16,float16,0,0.015146666516860327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,2,128,1,float16,fp8,0,0.017738666385412216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,4,128,1,float16,float16,0,0.015354666858911514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,4,128,1,float16,fp8,0,0.017786666750907898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,12,128,1,float16,float16,0,0.013162666310866674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,12,128,1,float16,fp8,0,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,1,128,1,float16,float16,0,0.014218666901191076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,1,128,1,float16,fp8,0,0.01640533283352852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,2,128,1,float16,float16,0,0.01421333352724711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,2,128,1,float16,fp8,0,0.01632533346613248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,4,128,1,float16,float16,0,0.014959999670584997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,4,128,1,float16,fp8,0,0.01655999943614006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,12,128,1,float16,float16,0,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,12,128,1,float16,fp8,0,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,1,128,1,float16,float16,0,0.01379199946920077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,1,128,1,float16,fp8,0,0.01600533351302147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,2,128,1,float16,float16,0,0.013898666948080063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,2,128,1,float16,fp8,0,0.01594666639963786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,4,128,1,float16,float16,0,0.014352000008026758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,4,128,1,float16,fp8,0,0.01626666635274887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,12,1,128,1,float16,float16,0,0.06346666812896729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,12,1,128,1,float16,fp8,0,0.072202667593956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,12,2,128,1,float16,float16,0,0.09099733829498291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,12,2,128,1,float16,fp8,0,0.09278933207194011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,12,128,1,float16,float16,0,0.0941493312517802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,12,4,128,1,float16,float16,0,0.12298132975896199
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,12,4,128,1,float16,fp8,0,0.14275200168291727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,12,128,1,float16,fp8,0,0.09427199761072795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,1,128,1,float16,float16,0,0.03090133269627889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,1,128,1,float16,fp8,0,0.03557866563399633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,2,128,1,float16,float16,0,0.04469866553942362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,2,128,1,float16,fp8,0,0.04586133360862732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,4,128,1,float16,float16,0,0.06619733572006226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,12,128,1,float16,float16,0,0.04924799998601278
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,12,128,1,float16,fp8,0,0.0430026650428772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,1,128,1,float16,fp8,0,0.025594666600227356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,4,128,1,float16,fp8,0,0.0637600024541219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,2,128,1,float16,float16,0,0.024512000381946564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,1,128,1,float16,float16,0,0.02223466585079829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,2,128,1,float16,fp8,0,0.02834133307139079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,4,128,1,float16,float16,0,0.029690665503342945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,12,128,1,float16,float16,0,0.02458133300145467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,4,128,1,float16,fp8,0,0.035258665680885315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,12,128,1,float16,fp8,0,0.02621866762638092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,1,128,1,float16,float16,0,0.015829333414634068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,1,128,1,float16,fp8,0,0.018437333405017853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,2,128,1,float16,float16,0,0.016970666746298473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,2,128,1,float16,fp8,0,0.0194560003777345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,4,128,1,float16,float16,0,0.01937066639463107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,4,128,1,float16,fp8,0,0.022645334402720135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,12,128,1,float16,fp8,0,0.01921066641807556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,12,128,1,float16,float16,0,0.018522666146357853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,1,128,1,float16,float16,0,0.015573333948850632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,1,128,1,float16,fp8,0,0.017621333400408428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,2,128,1,float16,float16,0,0.015306666493415833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,2,128,1,float16,fp8,0,0.018138666947682697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,4,128,1,float16,float16,0,0.016384000579516094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,4,128,1,float16,fp8,0,0.019402666638294857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,12,128,1,float16,fp8,0,0.014175999909639359
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,1,128,1,float16,float16,0,0.015119999647140503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,12,128,1,float16,float16,0,0.01350933313369751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,1,128,1,float16,fp8,0,0.017535999417304993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,2,128,1,float16,float16,0,0.014885333677132925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,2,128,1,float16,fp8,0,0.0173333336909612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,4,128,1,float16,float16,0,0.015087999403476715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,4,128,1,float16,fp8,0,0.017594666530688603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,12,128,1,float16,float16,0,0.013514666507641474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,12,128,1,float16,fp8,0,0.013962666193644205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,1,128,1,float16,float16,0,0.014122666170199713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,1,128,1,float16,fp8,0,0.01626666635274887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,2,128,1,float16,float16,0,0.014757333944241205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,2,128,1,float16,fp8,0,0.01628799984852473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,4,128,1,float16,float16,0,0.0141546664138635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,4,128,1,float16,fp8,0,0.016330666840076447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,12,128,1,float16,float16,0,0.013157332936922709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,12,128,1,float16,fp8,0,0.013610667238632837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,1,128,1,float16,float16,0,0.014106666048367819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,1,128,1,float16,fp8,0,0.016000000139077503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,2,128,1,float16,float16,0,0.013967999567588171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,2,128,1,float16,fp8,0,0.01590399940808614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,4,128,1,float16,float16,0,0.0138026662170887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,4,128,1,float16,fp8,0,0.016149333367745083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,12,128,1,float16,float16,0,0.012378666549921036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,12,128,1,float16,fp8,0,0.012730666746695837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,1,128,1,float16,float16,0,0.013983999689420065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,1,128,1,float16,fp8,0,0.015989333391189575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,2,128,1,float16,float16,0,0.013957332819700241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,2,128,1,float16,fp8,0,0.01599466676513354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,4,128,1,float16,float16,0,0.013584000368913015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,4,128,1,float16,fp8,0,0.016122666498025257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,12,1,128,1,float16,float16,0,0.03551999976237615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,12,1,128,1,float16,fp8,0,0.040591999888420105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,12,2,128,1,float16,float16,0,0.041936000188191734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,12,128,1,float16,float16,0,0.0678773323694865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,12,2,128,1,float16,fp8,0,0.046282668908437095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,12,128,1,float16,fp8,0,0.06161599854628245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,12,4,128,1,float16,float16,0,0.07178133229414622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,12,4,128,1,float16,fp8,0,0.06295999884605408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,1,128,1,float16,float16,0,0.022282667458057404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,1,128,1,float16,fp8,0,0.025733334322770435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,2,128,1,float16,float16,0,0.024351999163627625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,2,128,1,float16,fp8,0,0.028501334289709728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,4,128,1,float16,float16,0,0.03287466615438461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,12,128,1,float16,float16,0,0.03437866767247518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,4,128,1,float16,fp8,0,0.03537066777547201
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,12,128,1,float16,fp8,0,0.035530666510264076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,1,128,1,float16,float16,0,0.01573333392540614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,1,128,1,float16,fp8,0,0.018378666291634243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,4,128,1,float16,float16,0,0.021274665991465252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,2,128,1,float16,float16,0,0.016565332810084026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,2,128,1,float16,fp8,0,0.019493332753578823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,4,128,1,float16,fp8,0,0.022442666192849476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,12,128,1,float16,float16,0,0.0220320001244545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,12,128,1,float16,fp8,0,0.022704000274340313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,1,128,1,float16,float16,0,0.015082667271296183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,1,128,1,float16,fp8,0,0.017423999806245167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,2,128,1,float16,float16,0,0.015237333873907724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,2,128,1,float16,fp8,0,0.017370666066805523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,4,128,1,float16,float16,0,0.014981333166360855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,4,128,1,float16,fp8,0,0.016757333030303318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,12,128,1,float16,float16,0,0.016810666769742966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,12,128,1,float16,fp8,0,0.01732800031701724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,1,128,1,float16,float16,0,0.015066667149464289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,1,128,1,float16,fp8,0,0.017488000293572743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,2,128,1,float16,float16,0,0.014592000593741735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,2,128,1,float16,fp8,0,0.01716800034046173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,4,128,1,float16,float16,0,0.013429333766301474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,4,128,1,float16,fp8,0,0.013754667093356451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,12,128,1,float16,float16,0,0.013477332890033722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,12,128,1,float16,fp8,0,0.014064000298579534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,1,128,1,float16,float16,0,0.014362666755914688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,1,128,1,float16,fp8,0,0.016282666474580765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,2,128,1,float16,float16,0,0.014005333185195923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,2,128,1,float16,fp8,0,0.016143999993801117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,4,128,1,float16,float16,0,0.01310933381319046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,4,128,1,float16,fp8,0,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,12,128,1,float16,float16,0,0.013045333325862885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,12,128,1,float16,fp8,0,0.013647999614477158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,1,128,1,float16,float16,0,0.013888000200192133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,1,128,1,float16,fp8,0,0.016250666230916977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,2,128,1,float16,float16,0,0.013882666826248169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,2,128,1,float16,fp8,0,0.01597333326935768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,4,128,1,float16,float16,0,0.012757333616415659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,4,128,1,float16,fp8,0,0.013007999708255133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,12,128,1,float16,float16,0,0.013183999806642532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,12,128,1,float16,fp8,0,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,4,128,1,float16,float16,0,0.01231466606259346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,1,128,1,float16,float16,0,0.01403733342885971
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,1,128,1,float16,fp8,0,0.015909332782030106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,2,128,1,float16,float16,0,0.013861333330472311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,2,128,1,float16,fp8,0,0.015941333025693893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,4,128,1,float16,fp8,0,0.012634667257467905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,12,128,1,float16,float16,0,0.01202133297920227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,12,128,1,float16,fp8,0,0.012634667257467905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,1,128,1,float16,float16,0,0.01392000044385592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,1,128,1,float16,fp8,0,0.01590399940808614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,2,128,1,float16,float16,0,0.013514666507641474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,2,128,1,float16,fp8,0,0.01573333392540614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,4,128,1,float16,float16,0,0.012261333564917246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,4,128,1,float16,fp8,0,0.012341332932313284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,8,128,1,float16,float16,0,4.103359858194987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,8,128,1,float16,fp8,0,4.081770579020183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,8,1,128,1,float16,float16,0,6.995530446370442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,8,2,128,1,float16,float16,0,7.4839522043863935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,8,2,128,1,float16,fp8,0,7.196816126505534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,8,4,128,1,float16,float16,0,8.422159830729166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,8,4,128,1,float16,fp8,0,7.978309631347656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,8,1,128,1,float16,fp8,0,7.0694936116536455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,1,128,1,float16,float16,0,3.5557759602864585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,1,128,1,float16,fp8,0,3.5968427658081055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,2,128,1,float16,float16,0,3.7858667373657227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,2,128,1,float16,fp8,0,3.9047838846842446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,8,128,1,float16,float16,0,2.2015199661254883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,4,128,1,float16,float16,0,4.091504096984863
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,8,128,1,float16,fp8,0,2.1396053632100425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,4,128,1,float16,fp8,0,4.060453414916992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,1,128,1,float16,float16,0,1.8197120030721028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,8,128,1,float16,float16,0,4.0741119384765625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,8,128,1,float16,fp8,0,4.074282646179199
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,1,128,1,float16,fp8,0,1.9058666229248047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,2,128,1,float16,float16,0,1.9464853604634602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,2,128,1,float16,fp8,0,1.8988213539123535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,4,128,1,float16,float16,0,2.0716959635416665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,4,128,1,float16,fp8,0,2.0783626238505044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,8,128,1,float16,float16,0,2.1349546114603677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,8,128,1,float16,fp8,0,1.1390506426493328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,1,128,1,float16,float16,0,0.9511573314666748
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,8,128,1,float16,float16,0,1.1216586430867512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,1,128,1,float16,fp8,0,1.027898629506429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,8,128,1,float16,fp8,0,2.314810593922933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,2,128,1,float16,float16,0,1.0280053615570068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,2,128,1,float16,fp8,0,1.0046292940775554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,4,128,1,float16,float16,0,1.1040799617767334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,4,128,1,float16,fp8,0,1.1091466744740803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,8,128,1,float16,float16,0,1.1122132937113445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,8,128,1,float16,fp8,0,1.1363413333892822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,8,1,128,1,float16,float16,0,4.163205464680989
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,8,1,128,1,float16,fp8,0,4.157600084940593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,8,128,1,float16,float16,0,2.4476586977640786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,8,2,128,1,float16,float16,0,4.450954755147298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,8,128,1,float16,fp8,0,2.4625226656595864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,8,2,128,1,float16,fp8,0,4.148384094238281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,8,4,128,1,float16,float16,0,4.795520146687825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,8,4,128,1,float16,fp8,0,4.799706776936849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,1,128,1,float16,float16,0,2.068981329600016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,1,128,1,float16,fp8,0,2.113269329071045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,2,128,1,float16,float16,0,2.18613862991333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,2,128,1,float16,fp8,0,2.35317325592041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,4,128,1,float16,float16,0,2.4566292762756348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,4,128,1,float16,fp8,0,2.42739200592041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,8,128,1,float16,float16,0,2.475381374359131
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,8,128,1,float16,float16,0,1.2717173099517822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,1,128,1,float16,float16,0,1.0712107022603352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,8,128,1,float16,fp8,0,1.431872049967448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,8,128,1,float16,fp8,0,2.6225760777791343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,1,128,1,float16,fp8,0,1.163973331451416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,2,128,1,float16,float16,0,1.121029297510783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,4,128,1,float16,float16,0,1.2471733093261719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,2,128,1,float16,fp8,0,1.1857972939809163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,4,128,1,float16,fp8,0,1.3284586270650227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,8,128,1,float16,float16,0,1.2793973286946614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,8,128,1,float16,float16,0,0.6984053452809652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,8,128,1,float16,fp8,0,1.3510185877482097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,8,128,1,float16,fp8,0,0.7028586864471436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,1,128,1,float16,float16,0,0.5602933168411255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,1,128,1,float16,fp8,0,0.6125813325246176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,2,128,1,float16,float16,0,0.6120160023371378
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,2,128,1,float16,fp8,0,0.6282773415247599
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,4,128,1,float16,float16,0,0.6831146876017252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,4,128,1,float16,fp8,0,0.7192533016204834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,8,128,1,float16,float16,0,0.672917366027832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,8,128,1,float16,fp8,0,0.726037343343099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,8,1,128,1,float16,float16,0,2.8274399439493814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,8,1,128,1,float16,fp8,0,2.9544105529785156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,8,2,128,1,float16,float16,0,3.052037239074707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,8,128,1,float16,float16,0,1.7953920364379883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,8,2,128,1,float16,fp8,0,3.0484374364217124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,8,4,128,1,float16,float16,0,3.4631360371907554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,8,4,128,1,float16,fp8,0,3.561701456705729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,1,128,1,float16,float16,0,1.4532267252604167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,8,128,1,float16,fp8,0,1.9941226641337078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,1,128,1,float16,fp8,0,1.542319933573405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,2,128,1,float16,fp8,0,1.5727732976277669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,2,128,1,float16,float16,0,1.5825120608011882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,4,128,1,float16,float16,0,1.7871467272440593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,8,128,1,float16,float16,0,0.9255466461181641
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,4,128,1,float16,fp8,0,1.845370610555013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,8,128,1,float16,float16,0,1.7964587211608887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,1,128,1,float16,float16,0,0.7683253288269043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,8,128,1,float16,fp8,0,0.9871573448181152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,8,128,1,float16,fp8,0,1.975279966990153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,1,128,1,float16,fp8,0,0.8218932946523031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,2,128,1,float16,float16,0,0.8425652980804443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,2,128,1,float16,fp8,0,0.8190240065256754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,8,128,1,float16,float16,0,0.9531893730163574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,4,128,1,float16,float16,0,0.9177866776784261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,4,128,1,float16,fp8,0,0.9766186873118082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,8,128,1,float16,float16,0,0.5161653359731039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,8,128,1,float16,fp8,0,0.9892746607462565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,8,128,1,float16,fp8,0,0.5488693316777548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,1,128,1,float16,float16,0,0.4092746575673421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,1,128,1,float16,fp8,0,0.4330506722132365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,2,128,1,float16,float16,0,0.4588266611099243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,2,128,1,float16,fp8,0,0.46213332811991376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,4,128,1,float16,float16,0,0.5028746525446574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,4,128,1,float16,fp8,0,0.5427199999491373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,8,128,1,float16,float16,0,0.5136693318684896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,8,128,1,float16,fp8,0,0.5459626515706381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,8,1,128,1,float16,float16,0,3.7031307220458984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,8,1,128,1,float16,fp8,0,3.7770986557006836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,8,128,1,float16,float16,0,2.378416061401367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,8,2,128,1,float16,float16,0,4.167749404907227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,8,2,128,1,float16,fp8,0,3.9557813008626304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,8,128,1,float16,fp8,0,2.454949378967285
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,8,4,128,1,float16,float16,0,4.675637245178223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,8,4,128,1,float16,fp8,0,4.814149220784505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,1,128,1,float16,float16,0,1.8940800031026204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,1,128,1,float16,fp8,0,1.96123202641805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,2,128,1,float16,float16,0,2.059669335683187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,2,128,1,float16,fp8,0,2.212432066599528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,4,128,1,float16,float16,0,2.3301706314086914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,4,128,1,float16,fp8,0,2.4298133850097656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,8,128,1,float16,float16,0,2.4920053482055664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,8,128,1,float16,float16,0,1.2716586589813232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,8,128,1,float16,fp8,0,1.2921600341796875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,8,128,1,float16,fp8,0,2.4516852696736655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,1,128,1,float16,float16,0,0.9609386920928955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,1,128,1,float16,fp8,0,1.0384746392567952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,2,128,1,float16,float16,0,1.0504906972249348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,4,128,1,float16,fp8,0,1.2954613367716472
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,8,128,1,float16,float16,0,1.206602652867635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,2,128,1,float16,fp8,0,1.0417173703511555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,4,128,1,float16,float16,0,1.2190346717834473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,8,128,1,float16,float16,0,0.6526293357213339
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,8,128,1,float16,fp8,0,1.2849067052205403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,2,128,1,float16,float16,0,0.5449173450469971
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,2,128,1,float16,fp8,0,0.5571626822153727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,8,128,1,float16,fp8,0,0.7021066347757975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,1,128,1,float16,float16,0,0.506991982460022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,4,128,1,float16,fp8,0,0.6813813050587972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,1,128,1,float16,fp8,0,0.5515573422114054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,4,128,1,float16,float16,0,0.639413317044576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,8,128,1,float16,float16,0,0.6476320028305054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,8,128,1,float16,float16,0,0.3581013282140096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,8,128,1,float16,fp8,0,0.6806826591491699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,8,128,1,float16,fp8,0,0.39421331882476807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,1,128,1,float16,float16,0,0.27666666110356647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,1,128,1,float16,fp8,0,0.3048746585845947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,2,128,1,float16,float16,0,0.3124693234761556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,2,128,1,float16,fp8,0,0.32041066884994507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,4,128,1,float16,float16,0,0.34909868240356445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,4,128,1,float16,fp8,0,0.37620266278584796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,8,128,1,float16,float16,0,0.3630506594975789
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,8,128,1,float16,fp8,0,0.39346667130788165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,8,1,128,1,float16,float16,0,2.16101868947347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,8,1,128,1,float16,fp8,0,2.245685259501139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,8,2,128,1,float16,float16,0,2.398639996846517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,8,128,1,float16,float16,0,1.478981335957845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,8,2,128,1,float16,fp8,0,2.38484795888265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,8,4,128,1,float16,float16,0,2.821125348409017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,1,128,1,float16,float16,0,1.1024266878763835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,8,128,1,float16,fp8,0,1.67630402247111
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,8,4,128,1,float16,fp8,0,2.9705492655436196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,1,128,1,float16,fp8,0,1.2935893535614014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,2,128,1,float16,float16,0,1.2251946926116943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,2,128,1,float16,fp8,0,1.2202506860097249
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,4,128,1,float16,float16,0,1.436207930246989
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,8,128,1,float16,float16,0,0.7822453180948893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,4,128,1,float16,fp8,0,1.5369067192077637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,8,128,1,float16,float16,0,1.502336025238037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,8,128,1,float16,fp8,0,0.8292213280995687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,1,128,1,float16,fp8,0,0.6144800186157227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,1,128,1,float16,float16,0,0.5698506832122803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,2,128,1,float16,fp8,0,0.6501919825871786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,8,128,1,float16,fp8,0,1.5573867162068684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,2,128,1,float16,float16,0,0.650762677192688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,4,128,1,float16,float16,0,0.7571146488189697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,4,128,1,float16,fp8,0,0.8097493648529053
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,8,128,1,float16,float16,0,0.7615253130594889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,8,128,1,float16,float16,0,0.41881601015726727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,8,128,1,float16,fp8,0,0.8134240309397379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,8,128,1,float16,fp8,0,0.4456799825032552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,1,128,1,float16,float16,0,0.31220799684524536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,1,128,1,float16,fp8,0,0.3375093142191569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,2,128,1,float16,float16,0,0.35132265090942383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,2,128,1,float16,fp8,0,0.3614933490753174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,8,128,1,float16,fp8,0,0.45343466599782306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,4,128,1,float16,float16,0,0.4076160192489624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,4,128,1,float16,fp8,0,0.4317440191904704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,8,128,1,float16,float16,0,0.41681067148844403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,8,128,1,float16,float16,0,0.24134933948516846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,8,128,1,float16,fp8,0,0.2611466646194458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,1,128,1,float16,float16,0,0.17574934164683023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,1,128,1,float16,fp8,0,0.19274665911992392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,2,128,1,float16,float16,0,0.1993066668510437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,2,128,1,float16,fp8,0,0.19913599888483682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,8,128,1,float16,float16,0,0.24080532789230347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,4,128,1,float16,fp8,0,0.24764800071716309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,4,128,1,float16,float16,0,0.2361546754837036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,8,128,1,float16,fp8,0,0.26103466749191284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,8,1,128,1,float16,float16,0,2.0114240646362305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,8,1,128,1,float16,fp8,0,2.159221331278483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,8,2,128,1,float16,float16,0,2.377354621887207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,8,2,128,1,float16,fp8,0,2.2854560216267905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,8,128,1,float16,float16,0,1.52455472946167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,8,128,1,float16,fp8,0,1.607221285502116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,1,128,1,float16,float16,0,1.0259253184000652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,8,4,128,1,float16,float16,0,2.9399468104044595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,8,4,128,1,float16,fp8,0,3.131882667541504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,1,128,1,float16,fp8,0,1.1056533654530842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,2,128,1,float16,float16,0,1.1937546730041504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,2,128,1,float16,fp8,0,1.170639991760254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,4,128,1,float16,float16,0,1.5056373278299968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,4,128,1,float16,fp8,0,1.6027572949727376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,8,128,1,float16,float16,0,1.5017120043436687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,8,128,1,float16,float16,0,0.7849280039469401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,8,128,1,float16,fp8,0,1.5992213884989421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,8,128,1,float16,fp8,0,0.8589279651641846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,1,128,1,float16,float16,0,0.5369013150533041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,1,128,1,float16,fp8,0,0.5659146706263224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,2,128,1,float16,float16,0,0.6374773184458414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,2,128,1,float16,fp8,0,0.6165653467178345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,4,128,1,float16,float16,0,0.7638773123423258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,4,128,1,float16,fp8,0,0.8281973203023275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,8,128,1,float16,float16,0,0.7951093514760336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,8,128,1,float16,float16,0,0.4181813398996989
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,8,128,1,float16,fp8,0,0.8576160271962484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,8,128,1,float16,fp8,0,0.45178667704264325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,1,128,1,float16,float16,0,0.2905866702397664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,1,128,1,float16,fp8,0,0.3086133400599162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,2,128,1,float16,float16,0,0.33823998769124347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,2,128,1,float16,fp8,0,0.3349493344624837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,4,128,1,float16,float16,0,0.4106026490529378
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,4,128,1,float16,fp8,0,0.447434663772583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,8,128,1,float16,float16,0,0.4195359945297241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,8,128,1,float16,fp8,0,0.4519999821980794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,8,128,1,float16,float16,0,0.2363626758257548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,8,128,1,float16,fp8,0,0.25462933381398517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,1,128,1,float16,float16,0,0.15954666336377463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,2,128,1,float16,float16,0,0.19167999426523843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,1,128,1,float16,fp8,0,0.17124799887339273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,2,128,1,float16,fp8,0,0.19353065888086954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,4,128,1,float16,float16,0,0.2257066567738851
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,4,128,1,float16,fp8,0,0.24503467480341592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,8,128,1,float16,float16,0,0.23533866802851358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,8,128,1,float16,fp8,0,0.25494933128356934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,8,128,1,float16,float16,0,0.14071466525395712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,8,128,1,float16,fp8,0,0.15383467078208923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,1,128,1,float16,float16,0,0.092549333969752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,1,128,1,float16,fp8,0,0.10565333565076192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,2,128,1,float16,float16,0,0.10725866754849751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,2,128,1,float16,fp8,0,0.11642666657765706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,4,128,1,float16,float16,0,0.12371200323104858
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,4,128,1,float16,fp8,0,0.13801067074139914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,8,128,1,float16,float16,0,0.14275200168291727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,8,128,1,float16,fp8,0,0.15041066209475198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,8,1,128,1,float16,float16,0,1.234554688135783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,8,1,128,1,float16,fp8,0,1.2966559727986653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,8,2,128,1,float16,float16,0,1.434559981028239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,8,2,128,1,float16,fp8,0,1.4250292778015137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,8,128,1,float16,float16,0,0.9816853205362955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,8,4,128,1,float16,float16,0,1.8849600156148274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,1,128,1,float16,float16,0,0.6316853364308676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,8,128,1,float16,fp8,0,1.1016960144042969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,1,128,1,float16,fp8,0,0.6777013142903646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,8,4,128,1,float16,fp8,0,2.054528077443441
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,2,128,1,float16,float16,0,0.742474635442098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,2,128,1,float16,fp8,0,0.7388426462809244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,4,128,1,float16,float16,0,0.9584960142771403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,4,128,1,float16,fp8,0,1.0526986916859944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,8,128,1,float16,float16,0,1.0014932950337727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,8,128,1,float16,float16,0,0.5172106822331747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,8,128,1,float16,fp8,0,0.5707253217697144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,8,128,1,float16,fp8,0,1.0863839785257976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,1,128,1,float16,float16,0,0.3321760098139445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,1,128,1,float16,fp8,0,0.36157333850860596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,2,128,1,float16,float16,0,0.3991786638895671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,2,128,1,float16,fp8,0,0.4004640181859334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,4,128,1,float16,float16,0,0.50164266427358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,4,128,1,float16,fp8,0,0.5580000082651774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,8,128,1,float16,float16,0,0.27884799242019653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,8,128,1,float16,fp8,0,0.5695306857426962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,8,128,1,float16,float16,0,0.5209759871164957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,8,128,1,float16,fp8,0,0.31223465998967487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,1,128,1,float16,float16,0,0.18408532937367758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,1,128,1,float16,fp8,0,0.1990613341331482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,2,128,1,float16,float16,0,0.22221332788467407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,4,128,1,float16,fp8,0,0.3073173364003499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,2,128,1,float16,fp8,0,0.22355733315149942
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,4,128,1,float16,float16,0,0.27136532465616864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,8,128,1,float16,float16,0,0.2824479937553406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,8,128,1,float16,fp8,0,0.31141332785288495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,8,128,1,float16,float16,0,0.16154133280118307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,8,128,1,float16,fp8,0,0.17995200554529825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,1,128,1,float16,float16,0,0.10609599947929382
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,1,128,1,float16,fp8,0,0.11628266175587972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,2,128,1,float16,float16,0,0.12546666463216147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,2,128,1,float16,fp8,0,0.12917866309483847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,8,128,1,float16,fp8,0,0.17946134010950723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,4,128,1,float16,float16,0,0.15450132886568704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,4,128,1,float16,fp8,0,0.1698346734046936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,8,128,1,float16,float16,0,0.1607253352801005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,8,128,1,float16,float16,0,0.09272000193595886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,8,128,1,float16,fp8,0,0.10075733065605164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,1,128,1,float16,float16,0,0.06819733480612437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,1,128,1,float16,fp8,0,0.07683733105659485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,2,128,1,float16,float16,0,0.07337599992752075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,2,128,1,float16,fp8,0,0.07940266529719035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,4,128,1,float16,float16,0,0.08480532964070638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,4,128,1,float16,fp8,0,0.0946560005346934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,8,128,1,float16,float16,0,0.09246399998664856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,8,128,1,float16,fp8,0,0.10076799988746643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,8,1,128,1,float16,float16,0,1.2139999866485596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,8,1,128,1,float16,fp8,0,1.3108906745910645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,8,2,128,1,float16,float16,0,1.5495093663533528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,8,2,128,1,float16,fp8,0,1.4856534004211426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,8,128,1,float16,float16,0,1.0733973185221355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,8,4,128,1,float16,float16,0,2.095461368560791
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,8,128,1,float16,fp8,0,1.1889493465423584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,1,128,1,float16,float16,0,0.6215360164642334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,1,128,1,float16,fp8,0,0.6679680347442627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,8,4,128,1,float16,fp8,0,2.2924373944600425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,2,128,1,float16,float16,0,0.7986346880594889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,2,128,1,float16,fp8,0,0.7589706579844157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,4,128,1,float16,float16,0,1.0590026378631592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,4,128,1,float16,fp8,0,1.200170675913493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,8,128,1,float16,float16,0,1.0737706820170085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,8,128,1,float16,float16,0,0.5723573366800944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,8,128,1,float16,fp8,0,0.635258674621582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,8,128,1,float16,fp8,0,1.1813759803771973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,1,128,1,float16,float16,0,0.3304906686147054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,1,128,1,float16,fp8,0,0.35309334595998126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,2,128,1,float16,float16,0,0.41486398379007977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,2,128,1,float16,fp8,0,0.4030719995498657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,4,128,1,float16,float16,0,0.5463146766026815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,4,128,1,float16,fp8,0,0.6104106505711874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,8,128,1,float16,float16,0,0.565392017364502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,8,128,1,float16,float16,0,0.30379199981689453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,8,128,1,float16,fp8,0,0.6375573476155599
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,8,128,1,float16,fp8,0,0.33481065432230633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,1,128,1,float16,float16,0,0.17950934171676636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,1,128,1,float16,fp8,0,0.19648534059524536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,4,128,1,float16,fp8,0,0.3327466646830241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,2,128,1,float16,fp8,0,0.2192373275756836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,4,128,1,float16,float16,0,0.2912320097287496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,8,128,1,float16,fp8,0,0.1881386637687683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,2,128,1,float16,float16,0,0.22881599267323813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,8,128,1,float16,float16,0,0.3071733315785726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,8,128,1,float16,fp8,0,0.3311786651611328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,8,128,1,float16,float16,0,0.16717867056528726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,1,128,1,float16,float16,0,0.10030933221181233
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,1,128,1,float16,fp8,0,0.10821333527565002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,2,128,1,float16,float16,0,0.1275146702925364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,2,128,1,float16,fp8,0,0.12639466921488443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,4,128,1,float16,float16,0,0.16194666425387064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,4,128,1,float16,fp8,0,0.17880000670750937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,8,128,1,float16,float16,0,0.16752533117930093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,8,128,1,float16,fp8,0,0.1898933251698812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,8,128,1,float16,float16,0,0.09645866354306538
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,8,128,1,float16,fp8,0,0.10990933577219646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,1,128,1,float16,float16,0,0.06154666841030121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,1,128,1,float16,fp8,0,0.0705386648575465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,2,128,1,float16,float16,0,0.07065600156784058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,2,128,1,float16,fp8,0,0.07521600027879079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,8,128,1,float16,fp8,0,0.105813334385554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,4,128,1,float16,float16,0,0.08730133374532063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,4,128,1,float16,fp8,0,0.09215999643007915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,8,128,1,float16,float16,0,0.09892800450325012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,8,128,1,float16,float16,0,0.05004799862702688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,8,128,1,float16,fp8,0,0.05407999952634176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,2,128,1,float16,fp8,0,0.04653333127498627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,1,128,1,float16,float16,0,0.03855466594298681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,1,128,1,float16,fp8,0,0.0431573341290156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,2,128,1,float16,float16,0,0.04233600199222565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,4,128,1,float16,float16,0,0.0476693312327067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,8,128,1,float16,float16,0,0.048384000857671104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,4,128,1,float16,fp8,0,0.054416000843048096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,8,128,1,float16,fp8,0,0.05397866666316986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,8,1,128,1,float16,float16,0,0.8419786294301351
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,8,1,128,1,float16,fp8,0,0.8687733014424642
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,8,2,128,1,float16,float16,0,1.0410719712575276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,8,2,128,1,float16,fp8,0,1.01473601659139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,8,128,1,float16,float16,0,0.7943840026855469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,8,4,128,1,float16,float16,0,1.5136799812316895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,8,4,128,1,float16,fp8,0,1.663002649943034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,1,128,1,float16,float16,0,0.4407999912897746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,8,128,1,float16,fp8,0,0.8636586666107178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,1,128,1,float16,fp8,0,0.4540160099665324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,4,128,1,float16,float16,0,0.7658773263295492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,2,128,1,float16,float16,0,0.5563733180363973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,4,128,1,float16,fp8,0,0.8412906328837076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,8,128,1,float16,float16,0,0.8060159683227539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,8,128,1,float16,float16,0,0.41413335005442303
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,8,128,1,float16,fp8,0,0.8634560108184814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,8,128,1,float16,fp8,0,0.4554613431294759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,1,128,1,float16,float16,0,0.2340959906578064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,1,128,1,float16,fp8,0,0.24168533086776733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,2,128,1,float16,float16,0,0.3017599980036418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,2,128,1,float16,fp8,0,0.5218079884847006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,2,128,1,float16,fp8,0,0.28674133618672687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,4,128,1,float16,float16,0,0.4031253258387248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,4,128,1,float16,fp8,0,0.4365599950154622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,8,128,1,float16,float16,0,0.41619733969370526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,8,128,1,float16,fp8,0,0.45422931512196857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,8,128,1,float16,float16,0,0.22195200125376383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,8,128,1,float16,fp8,0,0.244704008102417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,1,128,1,float16,float16,0,0.13082133730252585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,1,128,1,float16,fp8,0,0.13820800185203552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,2,128,1,float16,float16,0,0.16384533047676086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,2,128,1,float16,fp8,0,0.15478400389353433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,4,128,1,float16,float16,0,0.2230506738026937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,4,128,1,float16,fp8,0,0.23507734139760336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,8,128,1,float16,float16,0,0.22272000710169473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,8,128,1,float16,fp8,0,0.24388800064722696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,8,128,1,float16,float16,0,0.11874133348464966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,8,128,1,float16,fp8,0,0.13934933145840964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,1,128,1,float16,float16,0,0.07273600002129872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,1,128,1,float16,fp8,0,0.07588266829649608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,2,128,1,float16,float16,0,0.09165866176287334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,2,128,1,float16,fp8,0,0.08399466673533122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,4,128,1,float16,float16,0,0.11808000008265178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,4,128,1,float16,fp8,0,0.12871999541918436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,8,128,1,float16,float16,0,0.11939733227094014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,8,128,1,float16,fp8,0,0.13987200458844504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,8,128,1,float16,float16,0,0.07041066884994507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,8,128,1,float16,fp8,0,0.07489066819349925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,1,128,1,float16,float16,0,0.04587199787298838
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,1,128,1,float16,fp8,0,0.04868799944718679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,2,128,1,float16,float16,0,0.051738664507865906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,2,128,1,float16,fp8,0,0.05412800113360087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,4,128,1,float16,float16,0,0.060032000144322716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,4,128,1,float16,fp8,0,0.06825600067774455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,8,128,1,float16,float16,0,0.07054933408896129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,8,128,1,float16,fp8,0,0.07415999968846639
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,8,128,1,float16,float16,0,0.04012800008058548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,8,128,1,float16,fp8,0,0.04459733267625173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,1,128,1,float16,float16,0,0.03270400067170461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,1,128,1,float16,fp8,0,0.0347626656293869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,2,128,1,float16,float16,0,0.03512533257404963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,2,128,1,float16,fp8,0,0.03804266701141993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,4,128,1,float16,float16,0,0.03938133269548416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,4,128,1,float16,fp8,0,0.043381333351135254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,8,128,1,float16,float16,0,0.03992533435424169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,8,128,1,float16,fp8,0,0.04452266792456309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,8,1,128,1,float16,float16,0,0.9274400075276693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,8,1,128,1,float16,fp8,0,0.9786400000254313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,8,2,128,1,float16,float16,0,1.2304373582204182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,8,128,1,float16,float16,0,0.9375680287679037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,8,2,128,1,float16,fp8,0,1.1798880100250244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,1,128,1,float16,float16,0,0.47837865352630615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,8,128,1,float16,fp8,0,1.0266186396280925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,8,4,128,1,float16,float16,0,1.7875040372212727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,1,128,1,float16,fp8,0,0.5046559969584147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,8,4,128,1,float16,fp8,0,2.005034605662028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,2,128,1,float16,float16,0,0.6477866570154825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,2,128,1,float16,fp8,0,0.6048159996668497
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,4,128,1,float16,float16,0,0.9210240046183268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,4,128,1,float16,fp8,0,1.021898667017619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,8,128,1,float16,fp8,0,1.0350293318430583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,1,128,1,float16,float16,0,0.2542613347371419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,8,128,1,float16,fp8,0,0.5512640078862509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,2,128,1,float16,fp8,0,0.31618134180704754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,1,128,1,float16,fp8,0,0.2654133240381877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,8,128,1,float16,float16,0,0.49539732933044434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,2,128,1,float16,float16,0,0.33853332201639813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,8,128,1,float16,float16,0,0.9286560217539469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,4,128,1,float16,float16,0,0.47118401527404785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,4,128,1,float16,fp8,0,0.5292533238728842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,8,128,1,float16,float16,0,0.4962986707687378
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,8,128,1,float16,float16,0,0.257914662361145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,8,128,1,float16,fp8,0,0.5618506669998169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,8,128,1,float16,fp8,0,0.2927893400192261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,1,128,1,float16,float16,0,0.13981866836547852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,1,128,1,float16,fp8,0,0.1481066644191742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,2,128,1,float16,float16,0,0.1832480033238729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,2,128,1,float16,fp8,0,0.17640533049901327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,4,128,1,float16,float16,0,0.24672534068425497
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,4,128,1,float16,fp8,0,0.2783466577529907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,8,128,1,float16,float16,0,0.25811199347178143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,8,128,1,float16,fp8,0,0.2972213427225749
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,8,128,1,float16,float16,0,0.14088533322016397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,8,128,1,float16,fp8,0,0.1618773341178894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,1,128,1,float16,float16,0,0.07785599927107494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,1,128,1,float16,fp8,0,0.0811413327852885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,2,128,1,float16,float16,0,0.10069866975148518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,8,128,1,float16,fp8,0,0.16224533319473267
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,2,128,1,float16,fp8,0,0.09796266754468282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,4,128,1,float16,float16,0,0.1338986655076345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,4,128,1,float16,fp8,0,0.15109333395957947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,8,128,1,float16,float16,0,0.13985600074132284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,8,128,1,float16,float16,0,0.07702399790287018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,8,128,1,float16,fp8,0,0.08965866764386494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,1,128,1,float16,float16,0,0.04601066807905833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,1,128,1,float16,fp8,0,0.051221330960591636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,2,128,1,float16,float16,0,0.051957334081331887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,2,128,1,float16,fp8,0,0.055173332492510475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,4,128,1,float16,float16,0,0.06621333460013072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,4,128,1,float16,fp8,0,0.0759200006723404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,8,128,1,float16,float16,0,0.07720000048478444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,8,128,1,float16,fp8,0,0.08912000060081482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,8,128,1,float16,float16,0,0.03806933263937632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,2,128,1,float16,fp8,0,0.03382933388153712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,8,128,1,float16,fp8,0,0.04228266576925913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,1,128,1,float16,float16,0,0.027813332776228588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,1,128,1,float16,fp8,0,0.030261332790056866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,2,128,1,float16,float16,0,0.031093334158261616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,4,128,1,float16,float16,0,0.03659199923276901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,4,128,1,float16,fp8,0,0.04053333401679993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,8,128,1,float16,float16,0,0.03826133410135905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,8,128,1,float16,fp8,0,0.042863999803860985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,8,128,1,float16,float16,0,0.03012799968322118
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,8,128,1,float16,fp8,0,0.034874667723973594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,1,128,1,float16,float16,0,0.025253333151340485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,1,128,1,float16,fp8,0,0.028010666370391846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,2,128,1,float16,float16,0,0.02743999908367793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,2,128,1,float16,fp8,0,0.030250666042168934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,4,128,1,float16,float16,0,0.030202666918436687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,4,128,1,float16,fp8,0,0.03382399926582972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,8,128,1,float16,float16,0,0.03031466652949651
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,8,128,1,float16,fp8,0,0.03473600000143051
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,8,1,128,1,float16,float16,0,0.6165706713994344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,8,1,128,1,float16,fp8,0,0.706554651260376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,8,2,128,1,float16,float16,0,0.9218560059865316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,8,2,128,1,float16,fp8,0,0.8929440180460612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,1,128,1,float16,float16,0,0.31995199124018353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,8,128,1,float16,float16,0,0.7992853323618571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,1,128,1,float16,fp8,0,0.3658133347829183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,8,128,1,float16,fp8,0,0.9529386361440023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,2,128,1,float16,float16,0,0.46801066398620605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,8,4,128,1,float16,float16,0,1.477178732554118
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,8,4,128,1,float16,fp8,0,1.7394453684488933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,4,128,1,float16,float16,0,0.7602453231811523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,8,128,1,float16,float16,0,0.41590933005015057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,8,128,1,float16,fp8,0,0.9598399798075358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,2,128,1,float16,fp8,0,0.4612266620000203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,4,128,1,float16,fp8,0,0.8913280169169108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,8,128,1,float16,float16,0,0.7930293083190918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,2,128,1,float16,float16,0,0.24235733350118002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,4,128,1,float16,float16,0,0.3914613326390584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,1,128,1,float16,fp8,0,0.19459732373555502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,8,128,1,float16,fp8,0,0.4978453318277995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,4,128,1,float16,fp8,0,0.45945600668589276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,8,128,1,float16,float16,0,0.21452800432840982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,8,128,1,float16,fp8,0,0.26204800605773926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,8,128,1,float16,fp8,0,0.49672532081604004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,2,128,1,float16,fp8,0,0.2456480065981547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,8,128,1,float16,float16,0,0.4140799840291341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,1,128,1,float16,float16,0,0.17294933398564658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,1,128,1,float16,float16,0,0.0958133339881897
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,2,128,1,float16,float16,0,0.1368053356806437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,1,128,1,float16,fp8,0,0.1090613305568695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,4,128,1,float16,float16,0,0.2034133275349935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,2,128,1,float16,fp8,0,0.13545067111651102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,8,128,1,float16,float16,0,0.21799467007319132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,8,128,1,float16,float16,0,0.11972266435623169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,4,128,1,float16,fp8,0,0.24510933955510458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,8,128,1,float16,fp8,0,0.2573653260866801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,8,128,1,float16,fp8,0,0.14604266484578451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,1,128,1,float16,float16,0,0.053344001372655235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,1,128,1,float16,fp8,0,0.06010666489601135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,8,128,1,float16,float16,0,0.11958400408426921
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,2,128,1,float16,float16,0,0.080485333998998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,2,128,1,float16,fp8,0,0.07958933214346568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,4,128,1,float16,float16,0,0.11065600315729777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,4,128,1,float16,fp8,0,0.13365866740544638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,8,128,1,float16,fp8,0,0.14523733655611673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,8,128,1,float16,float16,0,0.06685333450635274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,8,128,1,float16,fp8,0,0.08147733410199483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,1,128,1,float16,float16,0,0.03346666693687439
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,1,128,1,float16,fp8,0,0.03850133220354716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,2,128,1,float16,float16,0,0.03847466657559077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,2,128,1,float16,fp8,0,0.04298133154710134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,4,128,1,float16,float16,0,0.05589866638183594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,4,128,1,float16,fp8,0,0.06252266466617584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,8,128,1,float16,float16,0,0.06695466736952464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,8,128,1,float16,fp8,0,0.07714133461316426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,8,128,1,float16,float16,0,0.02996266633272171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,8,128,1,float16,fp8,0,0.0372533326347669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,2,128,1,float16,fp8,0,0.028714666763941448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,1,128,1,float16,float16,0,0.021221332252025604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,1,128,1,float16,fp8,0,0.025461333493391674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,2,128,1,float16,float16,0,0.023765332996845245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,4,128,1,float16,float16,0,0.029631999631722767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,4,128,1,float16,fp8,0,0.036661334335803986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,8,128,1,float16,float16,0,0.029930666089057922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,8,128,1,float16,fp8,0,0.03730133424202601
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,8,128,1,float16,float16,0,0.024245334168275196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,8,128,1,float16,fp8,0,0.03001066545645396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,1,128,1,float16,float16,0,0.020245333512624104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,1,128,1,float16,fp8,0,0.023445333043734234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,2,128,1,float16,float16,0,0.02092266579469045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,2,128,1,float16,fp8,0,0.025290665527184803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,4,128,1,float16,float16,0,0.023472001155217487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,4,128,1,float16,fp8,0,0.029866665601730347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,8,128,1,float16,float16,0,0.023936000963052113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,8,128,1,float16,fp8,0,0.030400000512599945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,8,128,1,float16,float16,0,0.02125866711139679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,8,128,1,float16,fp8,0,0.026858667532602947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,1,128,1,float16,float16,0,0.019530666371186573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,1,128,1,float16,fp8,0,0.023024000227451324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,2,128,1,float16,float16,0,0.019914666811625164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,4,128,1,float16,float16,0,0.020831999679406483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,2,128,1,float16,fp8,0,0.02342933416366577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,4,128,1,float16,fp8,0,0.026320000489552815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,8,128,1,float16,float16,0,0.021477334201335907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,8,128,1,float16,fp8,0,0.02699733277161916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,8,1,128,1,float16,float16,0,0.27432000637054443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,8,1,128,1,float16,fp8,0,0.3156320055325826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,8,2,128,1,float16,float16,0,0.40635732809702557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,1,128,1,float16,float16,0,0.14717866977055868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,8,2,128,1,float16,fp8,0,0.4108853340148926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,1,128,1,float16,fp8,0,0.17037866512934366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,8,128,1,float16,float16,0,0.395087997118632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,8,128,1,float16,fp8,0,0.46194132169087726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,8,4,128,1,float16,float16,0,0.7111946741739908
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,8,4,128,1,float16,fp8,0,0.8473440011342367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,2,128,1,float16,float16,0,0.22817067305246988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,2,128,1,float16,fp8,0,0.2201919953028361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,4,128,1,float16,float16,0,0.36318933963775635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,4,128,1,float16,fp8,0,0.43355735143025714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,8,128,1,float16,float16,0,0.39614399274190265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,8,128,1,float16,fp8,0,0.46057601769765216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,1,128,1,float16,float16,0,0.08161066472530365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,8,128,1,float16,float16,0,0.2038559913635254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,1,128,1,float16,fp8,0,0.09702933828035991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,8,128,1,float16,fp8,0,0.23921066522598267
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,2,128,1,float16,float16,0,0.12412800391515096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,2,128,1,float16,fp8,0,0.12342400352160136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,8,128,1,float16,fp8,0,0.23753599325815836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,8,128,1,float16,float16,0,0.11345600088437398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,4,128,1,float16,float16,0,0.19208000103632608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,8,128,1,float16,float16,0,0.20686399936676025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,4,128,1,float16,fp8,0,0.231930673122406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,2,128,1,float16,float16,0,0.07156800230344136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,8,128,1,float16,fp8,0,0.13428266843159994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,1,128,1,float16,float16,0,0.04384533564249674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,1,128,1,float16,fp8,0,0.05246399839719137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,2,128,1,float16,fp8,0,0.07318399846553802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,4,128,1,float16,float16,0,0.10428800185521443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,8,128,1,float16,float16,0,0.11294399698575337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,4,128,1,float16,fp8,0,0.12528000275293985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,8,128,1,float16,fp8,0,0.13398399949073792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,8,128,1,float16,float16,0,0.06132266422112783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,8,128,1,float16,fp8,0,0.07096000015735626
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,1,128,1,float16,float16,0,0.02770666778087616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,4,128,1,float16,fp8,0,0.05620799958705902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,1,128,1,float16,fp8,0,0.03297066688537598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,2,128,1,float16,float16,0,0.03296533226966858
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,2,128,1,float16,fp8,0,0.03839999934037527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,4,128,1,float16,float16,0,0.04952533543109894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,8,128,1,float16,float16,0,0.06157866617043813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,8,128,1,float16,fp8,0,0.06779199838638306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,8,128,1,float16,float16,0,0.027263998985290527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,8,128,1,float16,fp8,0,0.03437866767247518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,1,128,1,float16,float16,0,0.019061333189407986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,1,128,1,float16,fp8,0,0.021375998854637146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,2,128,1,float16,float16,0,0.0210506667693456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,2,128,1,float16,fp8,0,0.024682665864626568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,4,128,1,float16,float16,0,0.026373334228992462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,4,128,1,float16,fp8,0,0.03309866786003113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,8,128,1,float16,float16,0,0.027232001225153606
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,8,128,1,float16,fp8,0,0.034346667428811394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,8,128,1,float16,float16,0,0.020853333175182343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,8,128,1,float16,fp8,0,0.027258666853109997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,1,128,1,float16,float16,0,0.017456000049908955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,1,128,1,float16,fp8,0,0.019551999866962433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,2,128,1,float16,float16,0,0.018309333672126133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,2,128,1,float16,fp8,0,0.02160533269246419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,4,128,1,float16,float16,0,0.021216000119845074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,4,128,1,float16,fp8,0,0.02643733223279317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,8,128,1,float16,float16,0,0.021253332495689392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,8,128,1,float16,fp8,0,0.02699200063943863
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,8,128,1,float16,float16,0,0.018272000054518383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,8,128,1,float16,fp8,0,0.023333333432674408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,1,128,1,float16,float16,0,0.01693333312869072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,1,128,1,float16,fp8,0,0.01930133377512296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,2,128,1,float16,float16,0,0.01718933383623759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,2,128,1,float16,fp8,0,0.019925333559513092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,4,128,1,float16,float16,0,0.018122666825850803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,4,128,1,float16,fp8,0,0.022346665461858112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,8,128,1,float16,float16,0,0.01851733277241389
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,8,128,1,float16,fp8,0,0.023717333873112995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,8,128,1,float16,float16,0,0.016234666109085083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,8,128,1,float16,fp8,0,0.019578666736682255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,1,128,1,float16,float16,0,0.015568000574906668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,1,128,1,float16,fp8,0,0.017968000223239262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,2,128,1,float16,float16,0,0.01581866666674614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,2,128,1,float16,fp8,0,0.018522666146357853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,4,128,1,float16,float16,0,0.01623999948302905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,8,128,1,float16,float16,0,0.016208000481128693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,4,128,1,float16,fp8,0,0.020842666427294414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,8,128,1,float16,fp8,0,0.019989332805077236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,8,1,128,1,float16,float16,0,0.1425333321094513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,8,1,128,1,float16,fp8,0,0.1518880029519399
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,8,2,128,1,float16,float16,0,0.21149333318074545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,8,2,128,1,float16,fp8,0,0.21025067567825317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,1,128,1,float16,float16,0,0.0767626663049062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,8,128,1,float16,float16,0,0.20122667153676352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,8,128,1,float16,fp8,0,0.2235413392384847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,8,4,128,1,float16,float16,0,0.35540799299875897
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,1,128,1,float16,fp8,0,0.08680533369382222
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,8,4,128,1,float16,fp8,0,0.417738676071167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,2,128,1,float16,float16,0,0.12243733803431193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,2,128,1,float16,fp8,0,0.11628266175587972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,4,128,1,float16,float16,0,0.18886399269104004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,8,128,1,float16,float16,0,0.20016533136367798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,8,128,1,float16,float16,0,0.10943999886512756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,4,128,1,float16,fp8,0,0.2200053334236145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,8,128,1,float16,fp8,0,0.2230506738026937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,1,128,1,float16,float16,0,0.03926933308442434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,1,128,1,float16,fp8,0,0.045797333121299744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,4,128,1,float16,float16,0,0.10523200035095215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,8,128,1,float16,fp8,0,0.1239413321018219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,2,128,1,float16,float16,0,0.06897066533565521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,2,128,1,float16,fp8,0,0.06286933521429698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,4,128,1,float16,fp8,0,0.11616533001263936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,8,128,1,float16,float16,0,0.1109920044740041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,8,128,1,float16,fp8,0,0.12351466218630473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,8,128,1,float16,float16,0,0.059861332178115845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,2,128,1,float16,fp8,0,0.033861334125200905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,8,128,1,float16,fp8,0,0.06484800080458324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,1,128,1,float16,float16,0,0.025445332129796345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,1,128,1,float16,fp8,0,0.028933333853880566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,2,128,1,float16,float16,0,0.0301706666747729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,4,128,1,float16,float16,0,0.04422933359940847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,4,128,1,float16,fp8,0,0.04896533489227295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,8,128,1,float16,float16,0,0.060133333007494606
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,8,128,1,float16,fp8,0,0.06412800153096516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,8,128,1,float16,float16,0,0.025301332275072735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,8,128,1,float16,fp8,0,0.029824001093705494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,1,128,1,float16,float16,0,0.017322666943073273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,4,128,1,float16,fp8,0,0.029578665892283123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,1,128,1,float16,fp8,0,0.019519999623298645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,2,128,1,float16,float16,0,0.019695999721686046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,2,128,1,float16,fp8,0,0.023024000227451324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,4,128,1,float16,float16,0,0.02496533344189326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,8,128,1,float16,float16,0,0.025418666501839954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,8,128,1,float16,fp8,0,0.029951999584833782
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,8,128,1,float16,float16,0,0.019733333339293797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,8,128,1,float16,fp8,0,0.023077333966890972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,1,128,1,float16,float16,0,0.016261332978804905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,1,128,1,float16,fp8,0,0.018160000443458557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,2,128,1,float16,float16,0,0.016613333175579708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,2,128,1,float16,fp8,0,0.019727999965349834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,4,128,1,float16,float16,0,0.019354666272799175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,4,128,1,float16,fp8,0,0.022991999983787537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,8,128,1,float16,float16,0,0.01966399947802226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,8,128,1,float16,fp8,0,0.023007998863856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,8,128,1,float16,float16,0,0.016757333030303318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,8,128,1,float16,fp8,0,0.01933866615096728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,1,128,1,float16,float16,0,0.015002666662136713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,1,128,1,float16,fp8,0,0.01699200024207433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,2,128,1,float16,float16,0,0.015770666301250458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,2,128,1,float16,fp8,0,0.01801066721479098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,4,128,1,float16,float16,0,0.016890666137139004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,4,128,1,float16,fp8,0,0.019402666638294857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,8,128,1,float16,float16,0,0.01700266698996226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,1,128,1,float16,fp8,0,0.016197333733240765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,8,128,1,float16,fp8,0,0.019306667149066925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,8,128,1,float16,float16,0,0.014592000593741735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,8,128,1,float16,fp8,0,0.016842667013406754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,1,128,1,float16,float16,0,0.014117332796255747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,8,128,1,float16,float16,0,0.014554666976133982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,2,128,1,float16,float16,0,0.01471466695268949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,2,128,1,float16,fp8,0,0.016634666671355564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,4,128,1,float16,float16,0,0.014869333555301031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,4,128,1,float16,fp8,0,0.017994667092959087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,8,128,1,float16,fp8,0,0.016890666137139004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,8,128,1,float16,float16,0,0.014298666268587112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,8,128,1,float16,fp8,0,0.01621866722901662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,1,128,1,float16,float16,0,0.014069333672523499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,1,128,1,float16,fp8,0,0.015674666812022526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,2,128,1,float16,float16,0,0.014298666268587112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,2,128,1,float16,fp8,0,0.01618133361140887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,4,128,1,float16,float16,0,0.014490666488806406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,4,128,1,float16,fp8,0,0.016341333587964375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,8,128,1,float16,float16,0,0.014218666901191076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,8,128,1,float16,fp8,0,0.016186666985352833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,8,1,128,1,float16,float16,0,0.0754559983809789
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,8,1,128,1,float16,fp8,0,0.08558400472005208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,8,2,128,1,float16,float16,0,0.12184000015258789
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,8,2,128,1,float16,fp8,0,0.1160640021165212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,8,128,1,float16,float16,0,0.11482666929562886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,8,128,1,float16,fp8,0,0.12133866548538208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,8,4,128,1,float16,float16,0,0.18736000855763754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,1,128,1,float16,float16,0,0.04020266731580099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,8,4,128,1,float16,fp8,0,0.21813867489496866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,1,128,1,float16,fp8,0,0.04529599845409393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,2,128,1,float16,float16,0,0.06845866640408833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,8,128,1,float16,float16,0,0.11500266194343567
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,2,128,1,float16,fp8,0,0.06431999802589417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,8,128,1,float16,fp8,0,0.121370663245519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,4,128,1,float16,float16,0,0.10498133301734924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,4,128,1,float16,fp8,0,0.11643200119336446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,8,128,1,float16,float16,0,0.06259199976921082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,8,128,1,float16,fp8,0,0.06084799766540527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,1,128,1,float16,float16,0,0.02447466552257538
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,4,128,1,float16,float16,0,0.04693866769472758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,1,128,1,float16,fp8,0,0.02881066749493281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,2,128,1,float16,float16,0,0.030495998760064442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,2,128,1,float16,fp8,0,0.03367999941110611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,4,128,1,float16,fp8,0,0.04957866668701172
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,8,128,1,float16,float16,0,0.06284800171852112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,8,128,1,float16,fp8,0,0.06075733403364817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,8,128,1,float16,float16,0,0.02683199942111969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,8,128,1,float16,fp8,0,0.02898666759332021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,1,128,1,float16,float16,0,0.01711999997496605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,1,128,1,float16,fp8,0,0.01974933346112569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,2,128,1,float16,float16,0,0.019626667102177937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,2,128,1,float16,fp8,0,0.0229066660006841
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,4,128,1,float16,float16,0,0.024703999360402424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,4,128,1,float16,fp8,0,0.029498666524887085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,8,128,1,float16,float16,0,0.02666666607062022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,8,128,1,float16,fp8,0,0.02890666574239731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,2,128,1,float16,float16,0,0.016693333784739178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,8,128,1,float16,float16,0,0.017797333498795826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,8,128,1,float16,fp8,0,0.01924266666173935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,1,128,1,float16,float16,0,0.015482666591803232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,1,128,1,float16,fp8,0,0.018245333184798557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,2,128,1,float16,fp8,0,0.019802667200565338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,4,128,1,float16,float16,0,0.01923199991385142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,4,128,1,float16,fp8,0,0.022650666534900665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,8,128,1,float16,float16,0,0.01773333301146825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,8,128,1,float16,fp8,0,0.019130667050679524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,8,128,1,float16,float16,0,0.014975999792416891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,8,128,1,float16,fp8,0,0.015728000551462173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,4,128,1,float16,float16,0,0.01659199967980385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,1,128,1,float16,float16,0,0.015578666081031164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,1,128,1,float16,fp8,0,0.017322666943073273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,2,128,1,float16,float16,0,0.015370666980743408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,2,128,1,float16,fp8,0,0.01766933376590411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,4,128,1,float16,fp8,0,0.019152000546455383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,8,128,1,float16,float16,0,0.01498666654030482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,8,128,1,float16,fp8,0,0.01586666703224182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,8,128,1,float16,float16,0,0.013610667238632837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,8,128,1,float16,fp8,0,0.013983999689420065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,1,128,1,float16,float16,0,0.014896000425020853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,1,128,1,float16,fp8,0,0.01603200038274129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,2,128,1,float16,float16,0,0.014432000617186228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,2,128,1,float16,fp8,0,0.016303999970356624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,4,128,1,float16,float16,0,0.015210667004187902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,4,128,1,float16,fp8,0,0.016677333662907284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,8,128,1,float16,float16,0,0.013562666873137156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,8,128,1,float16,fp8,0,0.013989333063364029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,8,128,1,float16,float16,0,0.013178666432698568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,8,128,1,float16,fp8,0,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,1,128,1,float16,float16,0,0.014010666559139887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,1,128,1,float16,fp8,0,0.01594666639963786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,2,128,1,float16,float16,0,0.014303999642531076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,2,128,1,float16,fp8,0,0.016122666498025257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,4,128,1,float16,float16,0,0.01404800017674764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,4,128,1,float16,fp8,0,0.016437333077192307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,8,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,8,128,1,float16,fp8,0,0.012960000584522883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,8,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,8,128,1,float16,fp8,0,0.012805332740147909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,1,128,1,float16,float16,0,0.013621332744757334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,1,128,1,float16,fp8,0,0.015439999600251516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,2,128,1,float16,float16,0,0.013946666071812311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,2,128,1,float16,fp8,0,0.015935999651749928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,4,128,1,float16,float16,0,0.01404800017674764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,4,128,1,float16,fp8,0,0.01587733378012975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,8,128,1,float16,float16,0,0.01231466606259346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,8,128,1,float16,fp8,0,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,8,1,128,1,float16,float16,0,0.038922667503356934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,8,1,128,1,float16,fp8,0,0.0447573314110438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,8,2,128,1,float16,float16,0,0.0681279997030894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,8,2,128,1,float16,fp8,0,0.06344533463319142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,8,128,1,float16,float16,0,0.07512533167997996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,8,4,128,1,float16,float16,0,0.1069493293762207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,8,4,128,1,float16,fp8,0,0.11339199542999268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,8,128,1,float16,fp8,0,0.07248533268769582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,1,128,1,float16,float16,0,0.024879999458789825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,1,128,1,float16,fp8,0,0.02867199977238973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,2,128,1,float16,float16,0,0.029696000119050343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,2,128,1,float16,fp8,0,0.033600000043710075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,4,128,1,float16,float16,0,0.0458186666170756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,4,128,1,float16,fp8,0,0.04854399959246317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,8,128,1,float16,float16,0,0.07481599847475688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,8,128,1,float16,float16,0,0.03283733377854029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,8,128,1,float16,fp8,0,0.035045333206653595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,8,128,1,float16,fp8,0,0.07316799958546956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,1,128,1,float16,float16,0,0.017125333348910015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,1,128,1,float16,fp8,0,0.019786667078733444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,2,128,1,float16,float16,0,0.019637333850065868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,2,128,1,float16,fp8,0,0.022863999009132385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,4,128,1,float16,float16,0,0.02611733227968216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,8,128,1,float16,fp8,0,0.02257599929968516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,4,128,1,float16,fp8,0,0.028789333999156952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,1,128,1,float16,float16,0,0.01568000018596649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,8,128,1,float16,float16,0,0.03320533285538355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,8,128,1,float16,fp8,0,0.03550933301448822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,8,128,1,float16,float16,0,0.021029333273569744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,1,128,1,float16,fp8,0,0.01766933376590411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,2,128,1,float16,float16,0,0.016501333564519882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,2,128,1,float16,fp8,0,0.01934933289885521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,4,128,1,float16,float16,0,0.017498667041460674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,4,128,1,float16,fp8,0,0.019071999937295914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,8,128,1,float16,float16,0,0.021007999777793884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,8,128,1,float16,fp8,0,0.02275199939807256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,8,128,1,float16,float16,0,0.014885333677132925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,8,128,1,float16,fp8,0,0.015557333827018738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,1,128,1,float16,float16,0,0.015216000378131866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,1,128,1,float16,fp8,0,0.017184000462293625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,2,128,1,float16,float16,0,0.015413332730531693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,2,128,1,float16,fp8,0,0.017722666263580322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,4,128,1,float16,float16,0,0.014762666076421738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,4,128,1,float16,fp8,0,0.015493333339691162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,8,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,8,128,1,float16,fp8,0,0.01544533297419548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,8,128,1,float16,float16,0,0.013157332936922709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,8,128,1,float16,fp8,0,0.013845333208640417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,1,128,1,float16,float16,0,0.014394666999578476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,1,128,1,float16,fp8,0,0.016250666230916977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,2,128,1,float16,float16,0,0.014469332993030548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,2,128,1,float16,fp8,0,0.017269333203633625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,4,128,1,float16,float16,0,0.013370666652917862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,4,128,1,float16,fp8,0,0.013962666193644205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,8,128,1,float16,float16,0,0.013210666676362356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,8,128,1,float16,fp8,0,0.014064000298579534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,8,128,1,float16,float16,0,0.013365333278973898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,8,128,1,float16,fp8,0,0.013823999712864557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,1,128,1,float16,float16,0,0.013882666826248169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,1,128,1,float16,fp8,0,0.01584533353646596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,2,128,1,float16,float16,0,0.014064000298579534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,2,128,1,float16,fp8,0,0.01599466676513354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,4,128,1,float16,float16,0,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,4,128,1,float16,fp8,0,0.012991999586423239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,8,128,1,float16,float16,0,0.013338666409254074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,8,128,1,float16,fp8,0,0.013973332941532135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,8,128,1,float16,float16,0,0.012693333129088083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,8,128,1,float16,fp8,0,0.013408000270525614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,1,128,1,float16,float16,0,0.014005333185195923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,4,128,1,float16,fp8,0,0.0129120002190272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,1,128,1,float16,fp8,0,0.015392000476519266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,2,128,1,float16,float16,0,0.013861333330472311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,2,128,1,float16,fp8,0,0.01580799991885821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,4,128,1,float16,float16,0,0.012256000190973282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,8,128,1,float16,float16,0,0.013861333330472311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,8,128,1,float16,fp8,0,0.013477332890033722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,8,128,1,float16,float16,0,0.012223999947309494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,8,128,1,float16,fp8,0,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,1,128,1,float16,float16,0,0.013440000514189402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,1,128,1,float16,fp8,0,0.015562667200962702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,2,128,1,float16,float16,0,0.01370666672786077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,2,128,1,float16,fp8,0,0.016000000139077503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,4,128,1,float16,float16,0,0.0120319997270902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,4,128,1,float16,fp8,0,0.01249066616098086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,8,128,1,float16,float16,0,0.012122667084137598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,8,128,1,float16,fp8,0,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,8,1,128,1,float16,float16,0,0.024608001112937927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,8,1,128,1,float16,fp8,0,0.028384000062942505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,8,2,128,1,float16,float16,0,0.032933334509531655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,8,2,128,1,float16,fp8,0,0.03359466542800268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,8,4,128,1,float16,float16,0,0.06181333462397257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,8,128,1,float16,float16,0,0.04563199977080027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,8,128,1,float16,fp8,0,0.04717866579691569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,1,128,1,float16,float16,0,0.016949333250522614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,8,4,128,1,float16,fp8,0,0.06035733222961426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,1,128,1,float16,fp8,0,0.019493332753578823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,2,128,1,float16,float16,0,0.021381333470344543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,4,128,1,float16,float16,0,0.0328053335348765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,2,128,1,float16,fp8,0,0.022410665949185688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,4,128,1,float16,fp8,0,0.0349440003434817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,8,128,1,float16,float16,0,0.045738667249679565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,8,128,1,float16,fp8,0,0.047600001096725464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,8,128,1,float16,float16,0,0.027445333699385326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,8,128,1,float16,fp8,0,0.028949332733949024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,1,128,1,float16,float16,0,0.01594666639963786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,2,128,1,float16,fp8,0,0.015850666910409927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,1,128,1,float16,fp8,0,0.01785600061217944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,2,128,1,float16,float16,0,0.015024000157912573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,4,128,1,float16,float16,0,0.0210506667693456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,4,128,1,float16,fp8,0,0.022218666970729828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,8,128,1,float16,float16,0,0.027855999767780304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,8,128,1,float16,float16,0,0.018453333526849747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,8,128,1,float16,fp8,0,0.029274667302767437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,8,128,1,float16,fp8,0,0.01937599976857503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,1,128,1,float16,float16,0,0.015029333531856537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,1,128,1,float16,fp8,0,0.017125333348910015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,2,128,1,float16,float16,0,0.01349866638580958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,2,128,1,float16,fp8,0,0.01394133393963178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,4,128,1,float16,float16,0,0.014917333920796713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,4,128,1,float16,fp8,0,0.015935999651749928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,8,128,1,float16,float16,0,0.01844800015290578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,8,128,1,float16,fp8,0,0.018842666099468868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,8,128,1,float16,float16,0,0.01358933374285698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,8,128,1,float16,fp8,0,0.01402666668097178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,1,128,1,float16,float16,0,0.014074667046467463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,1,128,1,float16,fp8,0,0.016048000504573185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,2,128,1,float16,float16,0,0.01301866645614306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,2,128,1,float16,fp8,0,0.013568000247081121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,4,128,1,float16,float16,0,0.01331199953953425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,4,128,1,float16,fp8,0,0.013530666629473368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,8,128,1,float16,float16,0,0.013536000003417334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,8,128,1,float16,fp8,0,0.013936000565687815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,8,128,1,float16,float16,0,0.01341333364446958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,8,128,1,float16,fp8,0,0.013440000514189402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,1,128,1,float16,float16,0,0.013850666582584381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,1,128,1,float16,fp8,0,0.015861333658297855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,2,128,1,float16,float16,0,0.012453333785136541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,2,128,1,float16,fp8,0,0.01268799975514412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,4,128,1,float16,float16,0,0.013050666699806849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,4,128,1,float16,fp8,0,0.013909333695967993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,8,128,1,float16,float16,0,0.013370666652917862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,8,128,1,float16,fp8,0,0.013466666142145792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,8,128,1,float16,float16,0,0.012890666723251343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,8,128,1,float16,fp8,0,0.013584000368913015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,1,128,1,float16,float16,0,0.013503999759753546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,1,128,1,float16,fp8,0,0.015237333873907724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,2,128,1,float16,float16,0,0.012165332833925882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,2,128,1,float16,fp8,0,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,4,128,1,float16,float16,0,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,4,128,1,float16,fp8,0,0.012714666624863943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,8,128,1,float16,float16,0,0.012879999975363413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,8,128,1,float16,fp8,0,0.013541333377361298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,8,128,1,float16,float16,0,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,8,128,1,float16,fp8,0,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,1,128,1,float16,float16,0,0.013546666751305262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,1,128,1,float16,fp8,0,0.01573333392540614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,2,128,1,float16,float16,0,0.011898666620254517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,2,128,1,float16,fp8,0,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,4,128,1,float16,float16,0,0.012346666306257248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,4,128,1,float16,fp8,0,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,8,128,1,float16,float16,0,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,8,128,1,float16,fp8,0,0.012693333129088083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,8,128,1,float16,float16,0,0.012234666695197424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,8,128,1,float16,fp8,0,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,1,128,1,float16,float16,0,0.013370666652917862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,1,128,1,float16,fp8,0,0.015376000354687372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,2,128,1,float16,float16,0,0.011727999895811081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,2,128,1,float16,fp8,0,0.012448000411192576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,4,128,1,float16,float16,0,0.011813333878914515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,4,128,1,float16,fp8,0,0.012261333564917246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,8,128,1,float16,float16,0,0.012181332955757776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,8,128,1,float16,fp8,0,0.013834666460752487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,4,1,128,1,float16,float16,0,3.693056106567383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,4,128,1,float16,float16,0,2.3378772735595703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,1,128,1,float16,float16,0,1.8935093879699707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,4,1,128,1,float16,fp8,0,3.6644748051961265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,4,128,1,float16,fp8,0,2.410757382710775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,1,128,1,float16,fp8,0,1.9081226984659831
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,4,2,128,1,float16,float16,0,3.9351733525594077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,4,2,128,1,float16,fp8,0,3.804112116495768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,2,128,1,float16,float16,0,2.2209067344665527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,2,128,1,float16,fp8,0,1.965402603149414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,4,128,1,float16,float16,0,1.2250293095906575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,4,128,1,float16,float16,0,2.33350404103597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,4,128,1,float16,fp8,0,1.273370663324992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,1,128,1,float16,fp8,0,1.0661439895629883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,4,128,1,float16,fp8,0,2.4123147328694663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,1,128,1,float16,float16,0,1.002837340037028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,2,128,1,float16,float16,0,1.0729440053304036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,2,128,1,float16,fp8,0,1.0454400380452473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,4,128,1,float16,float16,0,0.6641973257064819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,4,128,1,float16,fp8,0,1.2721760272979736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,4,128,1,float16,float16,0,1.2277173201243083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,4,128,1,float16,fp8,0,0.7034933567047119
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,1,128,1,float16,float16,0,0.5419253508249918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,1,128,1,float16,fp8,0,0.5716053247451782
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,2,128,1,float16,float16,0,0.5830933252970377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,2,128,1,float16,fp8,0,0.590831995010376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,4,128,1,float16,float16,0,0.6644533475240072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,4,128,1,float16,fp8,0,0.7061920166015625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,4,1,128,1,float16,float16,0,2.1792160669962564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,4,1,128,1,float16,fp8,0,2.176581382751465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,4,128,1,float16,float16,0,1.439450740814209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,4,2,128,1,float16,float16,0,2.3122666676839194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,1,128,1,float16,float16,0,1.1202186743418376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,4,128,1,float16,fp8,0,1.530783971150716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,1,128,1,float16,fp8,0,1.1762719949086506
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,4,2,128,1,float16,fp8,0,2.2621706326802573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,2,128,1,float16,float16,0,1.2106133302052815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,2,128,1,float16,fp8,0,1.2219786643981934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,4,128,1,float16,float16,0,1.449600060780843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,4,128,1,float16,float16,0,0.7673333485921224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,4,128,1,float16,fp8,0,0.8482133547465006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,1,128,1,float16,float16,0,0.5984160105387369
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,4,128,1,float16,fp8,0,1.531786600748698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,1,128,1,float16,fp8,0,0.6238826513290405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,2,128,1,float16,float16,0,0.6563946803410848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,2,128,1,float16,fp8,0,0.6613706747690836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,4,128,1,float16,float16,0,0.7666026751200358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,4,128,1,float16,float16,0,0.43672001361846924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,4,128,1,float16,fp8,0,0.8306026458740234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,4,128,1,float16,fp8,0,0.4748479922612508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,1,128,1,float16,float16,0,0.3368213176727295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,2,128,1,float16,float16,0,0.37938666343688965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,1,128,1,float16,fp8,0,0.3477120002110799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,2,128,1,float16,fp8,0,0.38258667786916095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,4,128,1,float16,float16,0,0.43672001361846924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,4,128,1,float16,fp8,0,0.47066664695739746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,4,1,128,1,float16,float16,0,1.5577707290649414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,4,1,128,1,float16,fp8,0,1.568933327992757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,4,2,128,1,float16,float16,0,1.684666633605957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,4,128,1,float16,float16,0,1.0723199844360352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,4,2,128,1,float16,fp8,0,1.6892320315043132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,4,128,1,float16,fp8,0,1.1524266401926677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,1,128,1,float16,float16,0,0.7980213165283203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,1,128,1,float16,fp8,0,0.8243412971496582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,2,128,1,float16,float16,0,0.9041013717651367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,2,128,1,float16,fp8,0,0.892848014831543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,4,128,1,float16,float16,0,0.5811946789423624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,1,128,1,float16,fp8,0,0.4612533251444499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,4,128,1,float16,float16,0,1.075109322865804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,4,128,1,float16,fp8,0,0.6252053181330363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,4,128,1,float16,fp8,0,1.1422560214996338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,1,128,1,float16,float16,0,0.43822399775187176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,2,128,1,float16,float16,0,0.4997493426005046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,2,128,1,float16,fp8,0,0.48972801367441815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,4,128,1,float16,float16,0,0.5809119939804077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,4,128,1,float16,fp8,0,0.628496011098226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,4,128,1,float16,float16,0,0.3371146519978841
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,4,128,1,float16,fp8,0,0.3643840154012044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,1,128,1,float16,float16,0,0.25598400831222534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,1,128,1,float16,fp8,0,0.268069326877594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,2,128,1,float16,float16,0,0.28938132524490356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,2,128,1,float16,fp8,0,0.2797546585400899
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,4,128,1,float16,float16,0,0.33925867080688477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,4,128,1,float16,fp8,0,0.361407995223999
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,4,1,128,1,float16,float16,0,1.9740053812662761
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,4,1,128,1,float16,fp8,0,2.007061322530111
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,4,128,1,float16,float16,0,1.4781333605448406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,4,2,128,1,float16,float16,0,2.268592039744059
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,1,128,1,float16,float16,0,1.0061919689178467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,4,128,1,float16,fp8,0,1.5466400782267253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,4,2,128,1,float16,fp8,0,2.1736960411071777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,1,128,1,float16,fp8,0,1.0369866689046223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,2,128,1,float16,float16,0,1.2020959854125977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,4,128,1,float16,float16,0,1.4782613118489583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,2,128,1,float16,fp8,0,1.1340586344401042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,4,128,1,float16,fp8,0,1.588746706644694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,4,128,1,float16,float16,0,0.7566719849904379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,4,128,1,float16,fp8,0,0.8145493666330973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,2,128,1,float16,float16,0,0.6505493323008219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,1,128,1,float16,float16,0,0.5341813166936239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,1,128,1,float16,fp8,0,0.5549973249435425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,2,128,1,float16,fp8,0,0.6182719866434733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,4,128,1,float16,float16,0,0.7633386452992758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,4,128,1,float16,fp8,0,0.83460799853007
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,4,128,1,float16,float16,0,0.4181973139444987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,1,128,1,float16,float16,0,0.3014506697654724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,4,128,1,float16,fp8,0,0.4512639840443929
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,1,128,1,float16,fp8,0,0.3122239907582601
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,2,128,1,float16,float16,0,0.34619200229644775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,2,128,1,float16,fp8,0,0.3518933455149333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,4,128,1,float16,float16,0,0.4198506673177083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,4,128,1,float16,fp8,0,0.46426133314768475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,4,128,1,float16,float16,0,0.25034133593241376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,4,128,1,float16,fp8,0,0.27184534072875977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,1,128,1,float16,float16,0,0.17522132396697998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,4,128,1,float16,float16,0,0.25178666909535724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,1,128,1,float16,fp8,0,0.18376533190409342
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,2,128,1,float16,float16,0,0.20850133895874023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,2,128,1,float16,fp8,0,0.2108373244603475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,4,128,1,float16,fp8,0,0.27078400055567425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,4,1,128,1,float16,float16,0,1.179685354232788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,4,1,128,1,float16,fp8,0,1.2036853631337483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,4,2,128,1,float16,float16,0,1.3446985880533855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,4,128,1,float16,float16,0,0.9467199643452963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,4,2,128,1,float16,fp8,0,1.324191967646281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,1,128,1,float16,float16,0,0.6125973463058472
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,4,128,1,float16,fp8,0,1.021567980448405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,1,128,1,float16,fp8,0,0.6479520003000895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,2,128,1,float16,float16,0,0.7144052982330322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,2,128,1,float16,fp8,0,0.7190293471018473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,4,128,1,float16,float16,0,0.9467679659525553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,4,128,1,float16,float16,0,0.498149315516154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,4,128,1,float16,fp8,0,1.0131573677062988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,4,128,1,float16,fp8,0,0.5452586809794108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,1,128,1,float16,float16,0,0.3362666765848796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,1,128,1,float16,fp8,0,0.3551040093104045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,4,128,1,float16,fp8,0,0.543018658955892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,4,128,1,float16,float16,0,0.2813386718432109
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,2,128,1,float16,float16,0,0.4004160165786743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,2,128,1,float16,fp8,0,0.3933653434117635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,4,128,1,float16,float16,0,0.4973493417104085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,4,128,1,float16,fp8,0,0.31285866101582843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,1,128,1,float16,float16,0,0.19566933314005533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,1,128,1,float16,fp8,0,0.19885865847269693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,2,128,1,float16,float16,0,0.22735466559727988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,2,128,1,float16,fp8,0,0.23044800758361816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,4,128,1,float16,float16,0,0.28436267375946045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,4,128,1,float16,fp8,0,0.31100799640019733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,4,128,1,float16,float16,0,0.1665333310763041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,4,128,1,float16,fp8,0,0.17149333159128824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,1,128,1,float16,float16,0,0.12013333042462666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,1,128,1,float16,fp8,0,0.13131733735402426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,2,128,1,float16,float16,0,0.1323946714401245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,2,128,1,float16,fp8,0,0.1394773324330648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,4,128,1,float16,float16,0,0.16643733779589334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,4,128,1,float16,fp8,0,0.17536532878875732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,4,1,128,1,float16,float16,0,1.1362826824188232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,4,1,128,1,float16,fp8,0,1.177280028661092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,4,2,128,1,float16,float16,0,1.3972585995992024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,4,128,1,float16,float16,0,1.014250675837199
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,4,2,128,1,float16,fp8,0,1.3435306549072266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,1,128,1,float16,float16,0,0.5840959946314493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,4,128,1,float16,fp8,0,1.1215413411458333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,1,128,1,float16,fp8,0,0.6113760073979696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,2,128,1,float16,float16,0,0.7366613547007242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,2,128,1,float16,fp8,0,0.7015519936879476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,4,128,1,float16,float16,0,1.012384017308553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,4,128,1,float16,float16,0,0.5298666556676229
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,4,128,1,float16,fp8,0,1.1308053334554036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,4,128,1,float16,fp8,0,0.592416008313497
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,1,128,1,float16,float16,0,0.3213493426640828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,1,128,1,float16,fp8,0,0.3327893416086833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,2,128,1,float16,float16,0,0.396783987681071
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,2,128,1,float16,fp8,0,0.3779946565628052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,4,128,1,float16,float16,0,0.5266079902648926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,1,128,1,float16,fp8,0,0.18774400154749551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,4,128,1,float16,fp8,0,0.587173342704773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,2,128,1,float16,fp8,0,0.2167253295580546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,4,128,1,float16,float16,0,0.29242666562398273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,4,128,1,float16,float16,0,0.2868480086326599
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,4,128,1,float16,fp8,0,0.3237866759300232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,1,128,1,float16,float16,0,0.18419732650121054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,2,128,1,float16,float16,0,0.22114133834838867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,4,128,1,float16,fp8,0,0.3213333288828532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,4,128,1,float16,float16,0,0.1672160029411316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,4,128,1,float16,fp8,0,0.18281600872675577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,1,128,1,float16,float16,0,0.10738666852315266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,4,128,1,float16,fp8,0,0.1806346575419108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,1,128,1,float16,fp8,0,0.11478933691978455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,2,128,1,float16,float16,0,0.1253546675046285
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,2,128,1,float16,fp8,0,0.12760532895723978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,1,128,1,float16,fp8,0,0.0724373310804367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,4,128,1,float16,float16,0,0.16869866847991943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,4,128,1,float16,float16,0,0.08230933547019958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,4,128,1,float16,fp8,0,0.09360532959302266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,1,128,1,float16,float16,0,0.06646400193373363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,2,128,1,float16,float16,0,0.07051733136177063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,2,128,1,float16,fp8,0,0.07979199786980946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,4,128,1,float16,float16,0,0.08257066706816356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,4,1,128,1,float16,float16,0,0.7545493443806967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,4,128,1,float16,fp8,0,0.09371733665466309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,4,1,128,1,float16,fp8,0,0.7270507017771403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,4,2,128,1,float16,float16,0,0.9083519776662191
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,4,128,1,float16,float16,0,0.7070613702138265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,4,2,128,1,float16,fp8,0,0.8544747034708658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,4,128,1,float16,fp8,0,0.7745333512624105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,1,128,1,float16,float16,0,0.40185598532358807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,1,128,1,float16,fp8,0,0.38862399260203045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,2,128,1,float16,float16,0,0.49881601333618164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,2,128,1,float16,fp8,0,0.4496106704076131
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,4,128,1,float16,float16,0,0.7111252943674723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,4,128,1,float16,fp8,0,0.7693706353505453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,4,128,1,float16,float16,0,0.37812801202138263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,4,128,1,float16,fp8,0,0.4075466791788737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,1,128,1,float16,float16,0,0.22356800238291422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,1,128,1,float16,fp8,0,0.21422400077184042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,2,128,1,float16,float16,0,0.2738879919052124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,2,128,1,float16,fp8,0,0.247706671555837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,4,128,1,float16,float16,0,0.3789759874343872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,4,128,1,float16,float16,0,0.21013333400090536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,4,128,1,float16,fp8,0,0.40788265069325763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,4,128,1,float16,fp8,0,0.2261120080947876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,1,128,1,float16,float16,0,0.12366400162378947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,4,128,1,float16,float16,0,0.20985066890716553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,1,128,1,float16,fp8,0,0.12185600399971008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,2,128,1,float16,float16,0,0.15500266353289285
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,2,128,1,float16,fp8,0,0.14478933811187744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,4,128,1,float16,fp8,0,0.22531733910242716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,1,128,1,float16,fp8,0,0.07501866420110066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,4,128,1,float16,float16,0,0.11589866876602173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,4,128,1,float16,fp8,0,0.11708799997965495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,1,128,1,float16,float16,0,0.07551999886830647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,2,128,1,float16,float16,0,0.0844533344109853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,2,128,1,float16,fp8,0,0.08662399649620056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,4,128,1,float16,float16,0,0.11488533020019531
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,4,128,1,float16,fp8,0,0.11886399984359741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,4,128,1,float16,float16,0,0.06565866867701213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,4,128,1,float16,fp8,0,0.07063466807206471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,1,128,1,float16,float16,0,0.05418133238951365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,1,128,1,float16,fp8,0,0.054197331269582115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,4,128,1,float16,fp8,0,0.07051733136177063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,2,128,1,float16,float16,0,0.05747200051943461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,2,128,1,float16,fp8,0,0.05985599756240845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,4,128,1,float16,float16,0,0.06614399949709575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,4,1,128,1,float16,float16,0,0.7769066492716471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,4,1,128,1,float16,fp8,0,0.7788906892140707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,4,128,1,float16,float16,0,0.8282933235168457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,4,2,128,1,float16,float16,0,1.0562773545583088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,4,128,1,float16,fp8,0,0.9161813259124756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,1,128,1,float16,float16,0,0.4087573289871216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,4,2,128,1,float16,fp8,0,0.9353919823964437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,1,128,1,float16,fp8,0,0.4038240114847819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,2,128,1,float16,float16,0,0.5409546693166097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,2,128,1,float16,fp8,0,0.4909866650899251
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,4,128,1,float16,float16,0,0.8296426932017008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,4,128,1,float16,float16,0,0.4320586522420247
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,4,128,1,float16,fp8,0,0.4726399978001912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,4,128,1,float16,fp8,0,0.9112053712209066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,1,128,1,float16,fp8,0,0.21938665707906088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,1,128,1,float16,float16,0,0.2246560057004293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,2,128,1,float16,float16,0,0.2873599926630656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,2,128,1,float16,fp8,0,0.2651413281758626
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,4,128,1,float16,float16,0,0.43003201484680176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,4,128,1,float16,float16,0,0.22925333182017008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,4,128,1,float16,fp8,0,0.4766826629638672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,4,128,1,float16,fp8,0,0.2518720030784607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,1,128,1,float16,float16,0,0.12470933794975281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,1,128,1,float16,fp8,0,0.12531200051307678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,2,128,1,float16,float16,0,0.16513599952061972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,2,128,1,float16,fp8,0,0.15027733643849692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,4,128,1,float16,float16,0,0.2304906646410624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,4,128,1,float16,fp8,0,0.252128005027771
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,4,128,1,float16,float16,0,0.12744533022244772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,4,128,1,float16,fp8,0,0.13699733217557272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,1,128,1,float16,float16,0,0.07109866539637248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,1,128,1,float16,fp8,0,0.07274133463700612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,2,128,1,float16,float16,0,0.08726933598518372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,2,128,1,float16,fp8,0,0.0821973333756129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,4,128,1,float16,float16,0,0.12825066844622293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,4,128,1,float16,fp8,0,0.137061337629954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,4,128,1,float16,float16,0,0.06126933296521505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,4,128,1,float16,fp8,0,0.06390400230884552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,1,128,1,float16,float16,0,0.04292266567548116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,1,128,1,float16,fp8,0,0.04433066646258036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,2,128,1,float16,float16,0,0.04853333532810211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,2,128,1,float16,fp8,0,0.04943466683228811
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,4,128,1,float16,float16,0,0.059338668982187905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,4,128,1,float16,fp8,0,0.06423999865849812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,4,128,1,float16,float16,0,0.04785066843032837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,4,128,1,float16,fp8,0,0.051114668448766075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,1,128,1,float16,float16,0,0.03965333352486292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,1,128,1,float16,fp8,0,0.04002666721741358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,2,128,1,float16,float16,0,0.04252799848715464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,2,128,1,float16,fp8,0,0.0444106658299764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,4,128,1,float16,float16,0,0.04782933493455251
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,4,128,1,float16,fp8,0,0.050853331883748375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,4,1,128,1,float16,float16,0,0.5119573275248209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,4,1,128,1,float16,fp8,0,0.5207680066426595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,4,128,1,float16,float16,0,0.5837759971618652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,1,128,1,float16,float16,0,0.27611732482910156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,4,128,1,float16,fp8,0,0.6592959960301717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,4,2,128,1,float16,float16,0,0.6780373255411783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,1,128,1,float16,fp8,0,0.2792266607284546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,4,2,128,1,float16,fp8,0,0.6431039969126383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,2,128,1,float16,float16,0,0.3764053185780843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,2,128,1,float16,fp8,0,0.3388906717300415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,4,128,1,float16,float16,0,0.5864213307698568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,4,128,1,float16,fp8,0,0.6551040013631185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,4,128,1,float16,float16,0,0.3060373266537984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,1,128,1,float16,float16,0,0.15187199910481772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,4,128,1,float16,fp8,0,0.3444480101267497
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,1,128,1,float16,fp8,0,0.15557333827018738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,2,128,1,float16,float16,0,0.20357332626978555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,2,128,1,float16,fp8,0,0.1887893279393514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,4,128,1,float16,float16,0,0.30586665868759155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,1,128,1,float16,float16,0,0.08518933256467183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,4,128,1,float16,fp8,0,0.34441598256429035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,4,128,1,float16,float16,0,0.1664959987004598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,4,128,1,float16,fp8,0,0.1877760092417399
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,1,128,1,float16,fp8,0,0.08299200236797333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,2,128,1,float16,float16,0,0.11334932843844096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,2,128,1,float16,fp8,0,0.10813333590825398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,4,128,1,float16,float16,0,0.16554133097330728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,4,128,1,float16,fp8,0,0.18900799751281738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,4,128,1,float16,float16,0,0.09045333663622539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,4,128,1,float16,fp8,0,0.08794132868448894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,1,128,1,float16,float16,0,0.05178666611512502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,1,128,1,float16,fp8,0,0.053674668073654175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,2,128,1,float16,float16,0,0.059802666306495667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,4,128,1,float16,float16,0,0.04684266448020935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,2,128,1,float16,fp8,0,0.06322666505972545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,4,128,1,float16,float16,0,0.0848586658636729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,1,128,1,float16,fp8,0,0.03748266647259394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,4,128,1,float16,fp8,0,0.09533333778381348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,2,128,1,float16,fp8,0,0.04179200033346812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,4,128,1,float16,fp8,0,0.05376000205675761
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,1,128,1,float16,float16,0,0.03518400092919668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,2,128,1,float16,float16,0,0.0388373335202535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,4,128,1,float16,float16,0,0.047930667797724404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,4,128,1,float16,fp8,0,0.05392533540725708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,1,128,1,float16,fp8,0,0.03474666674931844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,4,128,1,float16,float16,0,0.038880000511805214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,4,128,1,float16,fp8,0,0.04358399907747904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,1,128,1,float16,float16,0,0.033200000723203026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,2,128,1,float16,float16,0,0.03472533325354258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,2,128,1,float16,fp8,0,0.03812800099452337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,4,128,1,float16,float16,0,0.039264000952243805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,4,128,1,float16,fp8,0,0.04362666606903076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,4,1,128,1,float16,float16,0,0.573082685470581
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,4,1,128,1,float16,fp8,0,0.5961546500523885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,1,128,1,float16,fp8,0,0.3123679955800374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,1,128,1,float16,float16,0,0.3003786603609721
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,2,128,1,float16,float16,0,0.43353601296742755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,4,128,1,float16,float16,0,0.7240479787190756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,4,128,1,float16,fp8,0,0.8287626902262369
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,4,2,128,1,float16,float16,0,0.8484160105387369
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,2,128,1,float16,fp8,0,0.39736000696818036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,4,2,128,1,float16,fp8,0,0.7671573162078857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,4,128,1,float16,float16,0,0.3710666497548421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,4,128,1,float16,float16,0,0.7216160297393799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,4,128,1,float16,fp8,0,0.8272426923116049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,4,128,1,float16,fp8,0,0.42551998297373456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,1,128,1,float16,float16,0,0.16291200121243796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,4,128,1,float16,float16,0,0.37112001578013104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,2,128,1,float16,float16,0,0.23996265729268393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,1,128,1,float16,float16,0,0.09261332949002583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,1,128,1,float16,fp8,0,0.17096533377965292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,4,128,1,float16,fp8,0,0.2286346753438314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,2,128,1,float16,float16,0,0.12923733393351236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,4,128,1,float16,fp8,0,0.4252053499221802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,2,128,1,float16,fp8,0,0.21530133485794067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,1,128,1,float16,fp8,0,0.09563733140627544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,4,128,1,float16,float16,0,0.1079306701819102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,4,128,1,float16,float16,0,0.19706666469573975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,2,128,1,float16,fp8,0,0.12227200468381245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,4,128,1,float16,float16,0,0.1962719957033793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,4,128,1,float16,fp8,0,0.22907733917236328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,2,128,1,float16,fp8,0,0.06356800099213918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,4,128,1,float16,fp8,0,0.11799466609954834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,1,128,1,float16,float16,0,0.05082133412361145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,1,128,1,float16,fp8,0,0.054511999090512596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,2,128,1,float16,float16,0,0.06574933230876923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,4,128,1,float16,float16,0,0.10745066404342651
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,4,128,1,float16,fp8,0,0.12074666221936543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,4,128,1,float16,fp8,0,0.05434666574001312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,4,128,1,float16,float16,0,0.04789866507053375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,1,128,1,float16,float16,0,0.030613332986831665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,1,128,1,float16,fp8,0,0.033733333150545754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,2,128,1,float16,float16,0,0.035589332381884255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,2,128,1,float16,fp8,0,0.03844800094763438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,4,128,1,float16,float16,0,0.04751466711362203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,4,128,1,float16,fp8,0,0.05458133419354757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,4,128,1,float16,float16,0,0.03500800083080927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,4,128,1,float16,fp8,0,0.04154133299986521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,1,128,1,float16,float16,0,0.027349332968393963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,4,128,1,float16,fp8,0,0.041146665811538696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,1,128,1,float16,fp8,0,0.030559999247392017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,2,128,1,float16,float16,0,0.0296426663796107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,1,128,1,float16,float16,0,0.025562666356563568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,2,128,1,float16,fp8,0,0.033376000821590424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,4,128,1,float16,float16,0,0.03551466763019562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,4,128,1,float16,float16,0,0.029893333713213604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,4,128,1,float16,fp8,0,0.034602666894594826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,1,128,1,float16,fp8,0,0.02807466685771942
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,2,128,1,float16,float16,0,0.027050666511058807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,2,128,1,float16,fp8,0,0.029946667452653248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,4,128,1,float16,float16,0,0.029706666866938274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,4,128,1,float16,fp8,0,0.034160000582536064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,4,1,128,1,float16,float16,0,0.4137386480967204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,4,1,128,1,float16,fp8,0,0.45243199666341144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,1,128,1,float16,float16,0,0.217466672261556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,4,128,1,float16,float16,0,0.6389333407084147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,4,128,1,float16,fp8,0,0.7632319927215576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,1,128,1,float16,fp8,0,0.24167466163635254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,4,2,128,1,float16,float16,0,0.6870826880137125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,2,128,1,float16,float16,0,0.33691732088724774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,2,128,1,float16,fp8,0,0.328874667485555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,4,2,128,1,float16,fp8,0,0.6274506648381551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,4,128,1,float16,float16,0,0.3257546623547872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,4,128,1,float16,float16,0,0.6372373501459757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,4,128,1,float16,fp8,0,0.7595360279083252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,1,128,1,float16,float16,0,0.11989333232243855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,1,128,1,float16,fp8,0,0.13407466808954874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,4,128,1,float16,fp8,0,0.3935786485671997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,2,128,1,float16,fp8,0,0.18125333388646445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,2,128,1,float16,float16,0,0.193615992863973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,4,128,1,float16,float16,0,0.32492266098658246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,4,128,1,float16,float16,0,0.17401599884033203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,4,128,1,float16,fp8,0,0.21050133307774863
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,4,128,1,float16,fp8,0,0.39238401254018146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,1,128,1,float16,float16,0,0.07029333213965099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,2,128,1,float16,fp8,0,0.10652800401051839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,1,128,1,float16,fp8,0,0.07690666615962982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,2,128,1,float16,float16,0,0.10636267066001892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,4,128,1,float16,float16,0,0.17468800147374472
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,4,128,1,float16,fp8,0,0.20957332849502563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,4,128,1,float16,float16,0,0.0942080020904541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,4,128,1,float16,fp8,0,0.10457600156466167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,1,128,1,float16,float16,0,0.03708266715208689
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,1,128,1,float16,fp8,0,0.04275733232498169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,2,128,1,float16,float16,0,0.050767997900644936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,2,128,1,float16,fp8,0,0.0536106675863266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,4,128,1,float16,float16,0,0.09445333480834961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,4,128,1,float16,fp8,0,0.10756267110506694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,4,128,1,float16,float16,0,0.0425600012143453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,2,128,1,float16,float16,0,0.028533334533373516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,1,128,1,float16,float16,0,0.0240639994541804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,4,128,1,float16,fp8,0,0.04953599969546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,1,128,1,float16,fp8,0,0.02887466549873352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,2,128,1,float16,fp8,0,0.03472533325354258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,4,128,1,float16,float16,0,0.04049066702524821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,4,128,1,float16,float16,0,0.028677334388097126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,4,128,1,float16,fp8,0,0.050069332122802734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,2,128,1,float16,fp8,0,0.029509333272775013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,4,128,1,float16,fp8,0,0.03639466563860575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,1,128,1,float16,float16,0,0.021407999098300934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,1,128,1,float16,fp8,0,0.02510933329661687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,2,128,1,float16,float16,0,0.023631999890009563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,4,128,1,float16,float16,0,0.028970666229724884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,4,128,1,float16,fp8,0,0.03617066641648611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,4,128,1,float16,float16,0,0.023765332996845245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,4,128,1,float16,fp8,0,0.029674666623274486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,1,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,1,128,1,float16,fp8,0,0.02386133372783661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,2,128,1,float16,float16,0,0.02062400057911873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,2,128,1,float16,fp8,0,0.0262719988822937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,4,128,1,float16,float16,0,0.023733332753181458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,4,128,1,float16,fp8,0,0.029658667743206024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,4,128,1,float16,float16,0,0.020330666253964107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,4,128,1,float16,fp8,0,0.025674665967623394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,1,128,1,float16,float16,0,0.019088000059127808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,1,128,1,float16,fp8,0,0.02250666668017705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,2,128,1,float16,float16,0,0.018917333334684372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,2,128,1,float16,fp8,0,0.02390933285156886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,4,128,1,float16,float16,0,0.02082666630546252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,4,128,1,float16,fp8,0,0.0258240004380544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,4,1,128,1,float16,float16,0,0.1950933337211609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,4,1,128,1,float16,fp8,0,0.2163146734237671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,4,128,1,float16,float16,0,0.3149120012919108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,4,2,128,1,float16,float16,0,0.3244160016377767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,1,128,1,float16,float16,0,0.1088106632232666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,4,128,1,float16,fp8,0,0.3730613390604655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,1,128,1,float16,fp8,0,0.12064533432324727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,4,2,128,1,float16,fp8,0,0.3072800040245056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,2,128,1,float16,fp8,0,0.16752533117930093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,2,128,1,float16,float16,0,0.16805867354075113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,4,128,1,float16,float16,0,0.16788800557454428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,4,128,1,float16,float16,0,0.3160586754480998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,4,128,1,float16,fp8,0,0.37094934781392414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,1,128,1,float16,fp8,0,0.06773333251476288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,4,128,1,float16,fp8,0,0.20020800828933716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,1,128,1,float16,float16,0,0.06148266792297363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,2,128,1,float16,float16,0,0.09879466891288757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,2,128,1,float16,fp8,0,0.09693333506584167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,4,128,1,float16,float16,0,0.1677280068397522
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,4,128,1,float16,fp8,0,0.19967466592788696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,4,128,1,float16,fp8,0,0.09836266438166301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,4,128,1,float16,float16,0,0.08848533034324646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,1,128,1,float16,float16,0,0.031194667021433514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,1,128,1,float16,fp8,0,0.037392000357309975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,2,128,1,float16,float16,0,0.0481333335240682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,2,128,1,float16,fp8,0,0.04842133323351542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,4,128,1,float16,float16,0,0.08847467104593913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,4,128,1,float16,fp8,0,0.10052800178527832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,4,128,1,float16,float16,0,0.03655466685692469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,2,128,1,float16,fp8,0,0.031167998909950256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,4,128,1,float16,fp8,0,0.04636266827583313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,1,128,1,float16,float16,0,0.020986666282018025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,1,128,1,float16,fp8,0,0.02481599897146225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,4,128,1,float16,fp8,0,0.03369066615899404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,2,128,1,float16,float16,0,0.026101333399613697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,4,128,1,float16,float16,0,0.037471999724706016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,4,128,1,float16,fp8,0,0.0462773342927297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,4,128,1,float16,float16,0,0.025850666066010792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,1,128,1,float16,float16,0,0.018298666924238205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,1,128,1,float16,fp8,0,0.021482666333516438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,2,128,1,float16,float16,0,0.021040000021457672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,2,128,1,float16,fp8,0,0.02565866708755493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,4,128,1,float16,float16,0,0.02606933315594991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,4,128,1,float16,fp8,0,0.03324799984693527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,4,128,1,float16,float16,0,0.020693333198626835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,4,128,1,float16,fp8,0,0.027045334378878277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,1,128,1,float16,float16,0,0.017322666943073273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,1,128,1,float16,fp8,0,0.019861333072185516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,2,128,1,float16,float16,0,0.01825599993268649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,2,128,1,float16,fp8,0,0.02232533444960912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,4,128,1,float16,float16,0,0.020799999435742695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,4,128,1,float16,fp8,0,0.027066667874654133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,4,128,1,float16,float16,0,0.018405333161354065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,4,128,1,float16,fp8,0,0.022229333718617756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,1,128,1,float16,float16,0,0.016127999871969223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,1,128,1,float16,fp8,0,0.01860800012946129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,2,128,1,float16,float16,0,0.01616000011563301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,2,128,1,float16,fp8,0,0.019621333728233974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,4,128,1,float16,float16,0,0.017594666530688603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,4,128,1,float16,fp8,0,0.022122666239738464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,4,128,1,float16,float16,0,0.01598400001724561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,4,128,1,float16,fp8,0,0.019674666225910187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,1,128,1,float16,float16,0,0.015637333194414776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,1,128,1,float16,fp8,0,0.018101333330074947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,2,128,1,float16,float16,0,0.015578666081031164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,2,128,1,float16,fp8,0,0.01913600042462349
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,4,128,1,float16,float16,0,0.016336000214020412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,4,128,1,float16,fp8,0,0.01953599974513054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,4,1,128,1,float16,float16,0,0.10442666212717693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,4,1,128,1,float16,fp8,0,0.11359999577204387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,4,128,1,float16,float16,0,0.1644159952799479
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,4,2,128,1,float16,float16,0,0.1772480010986328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,4,128,1,float16,fp8,0,0.18979199727376303
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,1,128,1,float16,float16,0,0.058602665861447654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,4,2,128,1,float16,fp8,0,0.1551146705945333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,1,128,1,float16,fp8,0,0.06186666587988535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,2,128,1,float16,float16,0,0.09718933701515198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,2,128,1,float16,fp8,0,0.08733866612116496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,4,128,1,float16,float16,0,0.16522666811943054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,1,128,1,float16,float16,0,0.029152000943819683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,4,128,1,float16,float16,0,0.08689066767692566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,4,128,1,float16,fp8,0,0.18900267283121744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,4,128,1,float16,fp8,0,0.09525332848230998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,1,128,1,float16,fp8,0,0.0330826664964358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,2,128,1,float16,float16,0,0.040405333042144775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,4,128,1,float16,fp8,0,0.04200000067551931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,1,128,1,float16,float16,0,0.019461333751678467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,2,128,1,float16,fp8,0,0.041519999504089355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,4,128,1,float16,float16,0,0.08646399776140849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,4,128,1,float16,fp8,0,0.09611733754475911
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,4,128,1,float16,float16,0,0.03608000030120214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,1,128,1,float16,fp8,0,0.02293866624434789
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,2,128,1,float16,float16,0,0.024453334510326385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,2,128,1,float16,fp8,0,0.027456000447273254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,1,128,1,float16,float16,0,0.016704000532627106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,4,128,1,float16,float16,0,0.03486400097608566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,4,128,1,float16,fp8,0,0.04182933270931244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,4,128,1,float16,float16,0,0.024533333877722423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,4,128,1,float16,fp8,0,0.029050665597120922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,1,128,1,float16,fp8,0,0.01956266661485036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,2,128,1,float16,float16,0,0.019413333386182785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,2,128,1,float16,fp8,0,0.02257599929968516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,4,128,1,float16,float16,0,0.02443733314673106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,1,128,1,float16,fp8,0,0.018058666338523228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,4,128,1,float16,fp8,0,0.029130667448043823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,4,128,1,float16,float16,0,0.019317333896954853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,4,128,1,float16,fp8,0,0.022543999056021374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,1,128,1,float16,float16,0,0.015637333194414776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,2,128,1,float16,float16,0,0.016906666258970898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,2,128,1,float16,fp8,0,0.019306667149066925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,4,128,1,float16,float16,0,0.019386666516462963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,4,128,1,float16,fp8,0,0.02252800017595291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,4,128,1,float16,float16,0,0.016048000504573185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,4,128,1,float16,fp8,0,0.01855466639002164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,1,128,1,float16,float16,0,0.014730667074521383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,1,128,1,float16,fp8,0,0.017594666530688603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,2,128,1,float16,float16,0,0.014938666174809137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,1,128,1,float16,float16,0,0.014117332796255747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,2,128,1,float16,fp8,0,0.016538667182127636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,4,128,1,float16,float16,0,0.016415999581416447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,2,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,4,128,1,float16,fp8,0,0.018506667266289394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,4,128,1,float16,float16,0,0.01545599972208341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,4,128,1,float16,float16,0,0.014021333307027817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,4,128,1,float16,fp8,0,0.016517333686351776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,1,128,1,float16,fp8,0,0.01613866661985715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,2,128,1,float16,fp8,0,0.016303999970356624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,4,128,1,float16,float16,0,0.014298666268587112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,4,128,1,float16,fp8,0,0.016229332735141117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,4,128,1,float16,fp8,0,0.016042667130629223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,1,128,1,float16,float16,0,0.013904000322024027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,1,128,1,float16,fp8,0,0.015930666277805965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,2,128,1,float16,float16,0,0.014117332796255747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,2,128,1,float16,fp8,0,0.01605333387851715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,4,128,1,float16,float16,0,0.013967999567588171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,4,128,1,float16,fp8,0,0.01607999950647354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,4,1,128,1,float16,float16,0,0.05996266504128774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,4,1,128,1,float16,fp8,0,0.05985066791375478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,4,2,128,1,float16,float16,0,0.09170666337013245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,4,128,1,float16,float16,0,0.09004799524943034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,4,128,1,float16,fp8,0,0.08386133114496867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,4,2,128,1,float16,fp8,0,0.08633066217104594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,1,128,1,float16,float16,0,0.029098667204380035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,1,128,1,float16,fp8,0,0.03301866600910822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,2,128,1,float16,float16,0,0.041519999504089355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,2,128,1,float16,fp8,0,0.04052799940109253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,4,128,1,float16,float16,0,0.04106133431196213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,4,128,1,float16,float16,0,0.08970133463541667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,4,128,1,float16,fp8,0,0.08555733164151509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,4,128,1,float16,fp8,0,0.04159466673930486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,1,128,1,float16,fp8,0,0.023024000227451324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,1,128,1,float16,float16,0,0.019717333217461903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,2,128,1,float16,float16,0,0.024346667031447094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,2,128,1,float16,fp8,0,0.027621333797772724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,4,128,1,float16,fp8,0,0.025397333006064098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,4,128,1,float16,float16,0,0.036650667587916054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,4,128,1,float16,fp8,0,0.04098666707674662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,4,128,1,float16,float16,0,0.022826666633288067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,1,128,1,float16,float16,0,0.01693333312869072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,1,128,1,float16,fp8,0,0.01953599974513054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,2,128,1,float16,float16,0,0.01937066639463107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,2,128,1,float16,fp8,0,0.022490667800108593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,4,128,1,float16,float16,0,0.02258666604757309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,4,128,1,float16,fp8,0,0.025263999899228413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,4,128,1,float16,float16,0,0.01736533393462499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,4,128,1,float16,fp8,0,0.018944000204404194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,1,128,1,float16,float16,0,0.015429332852363586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,1,128,1,float16,fp8,0,0.017680000513792038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,4,128,1,float16,fp8,0,0.01887999971707662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,2,128,1,float16,float16,0,0.016645333419243496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,2,128,1,float16,fp8,0,0.019130667050679524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,4,128,1,float16,float16,0,0.01740266631046931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,4,128,1,float16,float16,0,0.014815999815861383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,4,128,1,float16,fp8,0,0.015263999501864115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,1,128,1,float16,float16,0,0.014688000082969666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,1,128,1,float16,fp8,0,0.016634666671355564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,2,128,1,float16,float16,0,0.015072000523408255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,2,128,1,float16,fp8,0,0.016762666404247284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,4,128,1,float16,float16,0,0.014655999839305878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,4,128,1,float16,fp8,0,0.015360000232855478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,4,128,1,float16,float16,0,0.013056000073750814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,4,128,1,float16,fp8,0,0.013077333569526672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,1,128,1,float16,float16,0,0.014416000495354334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,1,128,1,float16,fp8,0,0.016309333344300587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,2,128,1,float16,float16,0,0.014298666268587112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,2,128,1,float16,fp8,0,0.016234666109085083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,4,128,1,float16,float16,0,0.012810666114091873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,4,128,1,float16,fp8,0,0.013514666507641474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,4,128,1,float16,float16,0,0.012469333906968435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,4,128,1,float16,fp8,0,0.012634667257467905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,1,128,1,float16,float16,0,0.014122666170199713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,1,128,1,float16,fp8,0,0.016202667107184727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,2,128,1,float16,float16,0,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,2,128,1,float16,fp8,0,0.015957333147525787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,4,128,1,float16,float16,0,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,4,128,1,float16,fp8,0,0.012981332838535309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,4,128,1,float16,float16,0,0.012085333466529846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,4,128,1,float16,fp8,0,0.012517333030700684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,1,128,1,float16,float16,0,0.014032000054915747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,1,128,1,float16,fp8,0,0.015941333025693893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,2,128,1,float16,float16,0,0.013861333330472311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,2,128,1,float16,fp8,0,0.015872000406185787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,4,128,1,float16,float16,0,0.011834666132926941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,4,128,1,float16,fp8,0,0.012608000387748083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,4,1,128,1,float16,float16,0,0.02898666759332021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,4,1,128,1,float16,fp8,0,0.03316266586383184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,4,128,1,float16,float16,0,0.04292266567548116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,4,2,128,1,float16,float16,0,0.04457066456476847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,4,128,1,float16,fp8,0,0.04790933430194855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,4,2,128,1,float16,fp8,0,0.04102399945259094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,1,128,1,float16,float16,0,0.01956266661485036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,1,128,1,float16,fp8,0,0.022991999983787537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,2,128,1,float16,float16,0,0.025989333788553875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,2,128,1,float16,fp8,0,0.026554666459560394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,4,128,1,float16,float16,0,0.04293333490689596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,4,128,1,float16,fp8,0,0.0476693312327067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,4,128,1,float16,float16,0,0.026149332523345947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,4,128,1,float16,fp8,0,0.028927999238173168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,1,128,1,float16,float16,0,0.016656000167131424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,1,128,1,float16,fp8,0,0.01942933350801468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,2,128,1,float16,float16,0,0.01762666677435239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,2,128,1,float16,fp8,0,0.018474667022625606
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,4,128,1,float16,float16,0,0.0260959987839063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,4,128,1,float16,fp8,0,0.02903466671705246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,4,128,1,float16,float16,0,0.017573333034912746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,4,128,1,float16,fp8,0,0.018917333334684372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,1,128,1,float16,float16,0,0.015439999600251516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,1,128,1,float16,fp8,0,0.018005333840847015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,2,128,1,float16,float16,0,0.014736000448465347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,2,128,1,float16,fp8,0,0.015247999380032221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,4,128,1,float16,float16,0,0.01746133342385292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,4,128,1,float16,fp8,0,0.018863999595244724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,4,128,1,float16,float16,0,0.014618666221698126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,2,128,1,float16,fp8,0,0.013461332768201828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,4,128,1,float16,fp8,0,0.01565333331624667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,1,128,1,float16,float16,0,0.014453332871198654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,1,128,1,float16,fp8,0,0.0164533331990242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,2,128,1,float16,float16,0,0.013349333157142004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,4,128,1,float16,float16,0,0.014698666830857595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,4,128,1,float16,float16,0,0.013151999562978745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,4,128,1,float16,fp8,0,0.015546667079130808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,4,128,1,float16,fp8,0,0.013760000467300415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,1,128,1,float16,float16,0,0.014170666535695394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,1,128,1,float16,fp8,0,0.016063999384641647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,2,128,1,float16,float16,0,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,2,128,1,float16,fp8,0,0.01312000056107839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,4,128,1,float16,float16,0,0.013093333691358566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,4,128,1,float16,fp8,0,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,4,128,1,float16,float16,0,0.012938667088747025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,4,128,1,float16,fp8,0,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,1,128,1,float16,float16,0,0.013861333330472311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,1,128,1,float16,fp8,0,0.01598400001724561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,2,128,1,float16,float16,0,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,2,128,1,float16,fp8,0,0.012608000387748083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,4,128,1,float16,float16,0,0.012608000387748083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,4,128,1,float16,fp8,0,0.013455999394257864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,4,128,1,float16,float16,0,0.012144000579913458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,4,128,1,float16,fp8,0,0.012698666503032049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,1,128,1,float16,float16,0,0.013850666582584381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,1,128,1,float16,fp8,0,0.015935999651749928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,2,128,1,float16,float16,0,0.011786667009194693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,2,128,1,float16,fp8,0,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,4,128,1,float16,float16,0,0.012191999703645706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,4,128,1,float16,fp8,0,0.01259200026591619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,4,128,1,float16,float16,0,0.012128000458081564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,4,128,1,float16,fp8,0,0.012538666526476542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,1,128,1,float16,float16,0,0.013872000078360239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,1,128,1,float16,fp8,0,0.015615999698638916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,2,128,1,float16,float16,0,0.011781333635250727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,2,128,1,float16,fp8,0,0.012527999778588613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,4,128,1,float16,float16,0,0.01180800050497055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,4,128,1,float16,fp8,0,0.01259200026591619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,4,1,128,1,float16,float16,0,0.02275199939807256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,4,1,128,1,float16,fp8,0,0.02276800076166789
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,4,128,1,float16,float16,0,0.032501332461833954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,4,2,128,1,float16,float16,0,0.032373333970705666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,4,128,1,float16,fp8,0,0.03509866694609324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,4,2,128,1,float16,fp8,0,0.033514666060606636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,1,128,1,float16,float16,0,0.015360000232855478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,1,128,1,float16,fp8,0,0.016069332758585613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,2,128,1,float16,float16,0,0.0210506667693456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,2,128,1,float16,fp8,0,0.022405333817005157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,4,128,1,float16,float16,0,0.032485333581765495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,4,128,1,float16,float16,0,0.02083733429511388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,4,128,1,float16,fp8,0,0.022453332940737408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,4,128,1,float16,fp8,0,0.03477866699298223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,1,128,1,float16,float16,0,0.013573333621025085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,1,128,1,float16,fp8,0,0.014053333550691605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,2,128,1,float16,float16,0,0.015130666395028433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,2,128,1,float16,fp8,0,0.015909332782030106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,4,128,1,float16,float16,0,0.020938667158285778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,4,128,1,float16,fp8,0,0.022389332453409832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,4,128,1,float16,float16,0,0.01504533365368843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,4,128,1,float16,fp8,0,0.015493333339691162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,1,128,1,float16,float16,0,0.012991999586423239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,1,128,1,float16,fp8,0,0.013552000125249227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,2,128,1,float16,float16,0,0.013744000345468521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,2,128,1,float16,fp8,0,0.0138026662170887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,4,128,1,float16,float16,0,0.014789332946141561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,4,128,1,float16,fp8,0,0.01562133307258288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,2,128,1,float16,fp8,0,0.013631999492645264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,4,128,1,float16,float16,0,0.01368533323208491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,4,128,1,float16,fp8,0,0.01357866699496905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,4,128,1,float16,fp8,0,0.013893333574136099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,4,128,1,float16,float16,0,0.013045333325862885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,1,128,1,float16,float16,0,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,1,128,1,float16,fp8,0,0.013354666531085968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,2,128,1,float16,float16,0,0.01303999995191892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,4,128,1,float16,float16,0,0.01312000056107839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,4,128,1,float16,fp8,0,0.013445333888133367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,1,128,1,float16,float16,0,0.012319999436537424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,1,128,1,float16,fp8,0,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,2,128,1,float16,float16,0,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,2,128,1,float16,fp8,0,0.012917333592971167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,4,128,1,float16,float16,0,0.01293333371480306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,4,128,1,float16,fp8,0,0.013434667140245438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,4,128,1,float16,float16,0,0.012874666601419449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,4,128,1,float16,fp8,0,0.0129120002190272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,1,128,1,float16,float16,0,0.011717333147923151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,1,128,1,float16,fp8,0,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,2,128,1,float16,float16,0,0.012138667205969492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,2,128,1,float16,fp8,0,0.012543999900420507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,4,128,1,float16,float16,0,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,4,128,1,float16,fp8,0,0.012815999488035837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,4,128,1,float16,float16,0,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,4,128,1,float16,fp8,0,0.012805332740147909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,1,128,1,float16,float16,0,0.011658667276302973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,1,128,1,float16,fp8,0,0.012298667182525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,2,128,1,float16,float16,0,0.011893333246310553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,2,128,1,float16,fp8,0,0.012608000387748083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,4,128,1,float16,float16,0,0.012341332932313284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,4,128,1,float16,fp8,0,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,4,128,1,float16,float16,0,0.01181866725285848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,4,128,1,float16,fp8,0,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,1,128,1,float16,float16,0,0.011626667032639185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,1,128,1,float16,fp8,0,0.012181332955757776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,2,128,1,float16,float16,0,0.011701333026091257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,2,128,1,float16,fp8,0,0.01249066616098086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,4,128,1,float16,float16,0,0.011866666376590729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,4,128,1,float16,fp8,0,0.012378666549921036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,2,2,128,1,float16,float16,0,1.1938292980194092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,2,1,128,1,float16,float16,0,1.9479519526163738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,2,2,128,1,float16,fp8,0,1.1445386409759521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,2,1,128,1,float16,float16,0,1.0265920162200928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,2,1,128,1,float16,fp8,0,1.9654399553934734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,2,1,128,1,float16,fp8,0,1.0527466932932537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,2,2,128,1,float16,float16,0,1.1826026439666748
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,2,2,128,1,float16,float16,0,0.6471733252207438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,2,2,128,1,float16,fp8,0,1.1488213539123535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,2,2,128,1,float16,fp8,0,0.6464159886042277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,2,1,128,1,float16,float16,0,0.5629813273747762
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,2,1,128,1,float16,fp8,0,0.5876906712849935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,2,2,128,1,float16,float16,0,0.6431680123011271
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,2,2,128,1,float16,float16,0,0.39979732036590576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,2,2,128,1,float16,fp8,0,0.6480000019073486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,2,2,128,1,float16,fp8,0,0.4020693302154541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,2,1,128,1,float16,float16,0,0.3311573266983032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,2,1,128,1,float16,fp8,0,0.34721601009368896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,2,2,128,1,float16,float16,0,0.4004319906234741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,2,2,128,1,float16,fp8,0,0.4009866714477539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,2,1,128,1,float16,float16,0,1.1669920285542805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,2,2,128,1,float16,float16,0,0.7454240322113037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,2,1,128,1,float16,fp8,0,1.187498648961385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,2,2,128,1,float16,fp8,0,0.7150986989339193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,2,1,128,1,float16,float16,0,0.6269866625467936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,2,1,128,1,float16,fp8,0,0.6525546709696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,2,2,128,1,float16,float16,0,0.42590399583180744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,2,2,128,1,float16,fp8,0,0.7150239944458008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,2,2,128,1,float16,float16,0,0.7344746589660645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,2,2,128,1,float16,fp8,0,0.41997865835825604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,2,1,128,1,float16,float16,0,0.36418668429056805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,2,1,128,1,float16,fp8,0,0.3765173355738322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,2,2,128,1,float16,float16,0,0.42683200041453045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,2,2,128,1,float16,fp8,0,0.4182080030441284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,2,2,128,1,float16,float16,0,0.25731732447942096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,2,2,128,1,float16,fp8,0,0.25038933753967285
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,2,1,128,1,float16,float16,0,0.22500799099604288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,2,1,128,1,float16,fp8,0,0.24223999182383218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,2,2,128,1,float16,float16,0,0.25519466400146484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,2,2,128,1,float16,fp8,0,0.25120000044504803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,2,1,128,1,float16,float16,0,0.8516533374786377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,2,1,128,1,float16,fp8,0,0.8703040281931559
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,2,2,128,1,float16,float16,0,0.5573973258336385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,2,2,128,1,float16,fp8,0,0.5404800176620483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,2,1,128,1,float16,float16,0,0.4678879976272583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,2,1,128,1,float16,fp8,0,0.4923199812571208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,2,1,128,1,float16,float16,0,0.27448532978693646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,2,2,128,1,float16,float16,0,0.5574346780776978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,2,2,128,1,float16,fp8,0,0.5397706826527914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,2,2,128,1,float16,float16,0,0.33029866218566895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,2,2,128,1,float16,fp8,0,0.32200533151626587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,2,1,128,1,float16,fp8,0,0.2815999984741211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,2,2,128,1,float16,float16,0,0.3311466574668884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,2,2,128,1,float16,fp8,0,0.3259893258412679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,2,2,128,1,float16,float16,0,0.18652800718943277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,2,2,128,1,float16,fp8,0,0.1932906707127889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,2,1,128,1,float16,float16,0,0.1729653278986613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,2,1,128,1,float16,fp8,0,0.18780267238616943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,2,2,128,1,float16,float16,0,0.185098667939504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,2,2,128,1,float16,fp8,0,0.19324266910552979
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,2,1,128,1,float16,float16,0,1.1018400192260742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,2,2,128,1,float16,float16,0,0.7216693560282389
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,2,1,128,1,float16,fp8,0,1.1192320187886555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,2,2,128,1,float16,fp8,0,0.6868906815846761
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,2,1,128,1,float16,float16,0,0.5822240114212036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,2,1,128,1,float16,fp8,0,0.6124853293100992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,2,2,128,1,float16,float16,0,0.7082719802856445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,2,2,128,1,float16,float16,0,0.4095199902852376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,2,2,128,1,float16,fp8,0,0.6870613098144531
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,2,2,128,1,float16,fp8,0,0.3926986853281657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,2,1,128,1,float16,float16,0,0.33178667227427167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,2,1,128,1,float16,fp8,0,0.34410667419433594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,2,2,128,1,float16,float16,0,0.39537068208058673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,2,2,128,1,float16,fp8,0,0.39420799414316815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,2,2,128,1,float16,float16,0,0.24582932392756143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,2,2,128,1,float16,fp8,0,0.23809067408243814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,2,1,128,1,float16,float16,0,0.19779199361801147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,2,1,128,1,float16,fp8,0,0.20881599187850952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,2,2,128,1,float16,float16,0,0.24673600991566977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,2,2,128,1,float16,fp8,0,0.2371306618054708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,2,2,128,1,float16,float16,0,0.1312266687552134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,2,2,128,1,float16,fp8,0,0.13979199528694153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,2,1,128,1,float16,float16,0,0.1188159982363383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,2,1,128,1,float16,fp8,0,0.1291306714216868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,2,2,128,1,float16,float16,0,0.13032000263532004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,2,2,128,1,float16,fp8,0,0.14004266262054443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,2,1,128,1,float16,float16,0,0.7185493310292562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,2,1,128,1,float16,fp8,0,0.659173329671224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,2,2,128,1,float16,float16,0,0.4856479962666829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,2,2,128,1,float16,fp8,0,0.4290826718012492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,2,1,128,1,float16,float16,0,0.3859306573867798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,2,1,128,1,float16,fp8,0,0.36190398534138996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,2,2,128,1,float16,float16,0,0.4804533322652181
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,2,2,128,1,float16,fp8,0,0.43137065569559735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,2,2,128,1,float16,float16,0,0.28538666168848675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,2,2,128,1,float16,fp8,0,0.2480693260828654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,2,1,128,1,float16,float16,0,0.22363199790318808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,2,1,128,1,float16,fp8,0,0.20921599864959717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,2,2,128,1,float16,float16,0,0.27933333317438763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,2,2,128,1,float16,fp8,0,0.2479520042737325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,2,2,128,1,float16,float16,0,0.16727999846140543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,2,2,128,1,float16,fp8,0,0.14457600315411887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,2,1,128,1,float16,float16,0,0.1317759950955709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,2,1,128,1,float16,fp8,0,0.12703999876976013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,2,2,128,1,float16,float16,0,0.16566399733225504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,2,2,128,1,float16,fp8,0,0.14332800110181174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,2,2,128,1,float16,float16,0,0.10231467088063557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,2,2,128,1,float16,fp8,0,0.1006666620572408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,2,1,128,1,float16,float16,0,0.09593066573143005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,2,1,128,1,float16,fp8,0,0.09171733260154724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,2,2,128,1,float16,float16,0,0.10288000106811523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,2,2,128,1,float16,fp8,0,0.10033067067464192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,2,1,128,1,float16,float16,0,0.7008053461710612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,2,2,128,1,float16,float16,0,0.5072640180587769
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,2,1,128,1,float16,fp8,0,0.671125332514445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,2,2,128,1,float16,fp8,0,0.44882134596506756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,2,1,128,1,float16,float16,0,0.37482666969299316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,2,1,128,1,float16,fp8,0,0.36188264687856037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,2,2,128,1,float16,float16,0,0.5088906685511271
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,2,2,128,1,float16,fp8,0,0.2483146588007609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,2,2,128,1,float16,float16,0,0.27612266937891644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,2,2,128,1,float16,fp8,0,0.4421493212381999
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,2,1,128,1,float16,float16,0,0.21574934323628744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,2,1,128,1,float16,fp8,0,0.20664000511169434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,2,2,128,1,float16,float16,0,0.2774453361829122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,2,2,128,1,float16,fp8,0,0.24985599517822266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,2,2,128,1,float16,float16,0,0.16774932543436685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,2,2,128,1,float16,float16,0,0.17069333791732788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,2,2,128,1,float16,fp8,0,0.14520532886187235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,2,2,128,1,float16,fp8,0,0.1477120021979014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,2,1,128,1,float16,float16,0,0.12170132994651794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,2,1,128,1,float16,fp8,0,0.11899200081825256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,2,2,128,1,float16,float16,0,0.08379733562469482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,2,2,128,1,float16,fp8,0,0.08055999875068665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,2,2,128,1,float16,fp8,0,0.08204799890518188
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,2,1,128,1,float16,float16,0,0.072202667593956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,2,1,128,1,float16,fp8,0,0.07292800148328145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,2,2,128,1,float16,float16,0,0.08258133133252461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,2,2,128,1,float16,float16,0,0.07189866900444031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,2,2,128,1,float16,fp8,0,0.07315200070540111
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,2,1,128,1,float16,float16,0,0.06710400183995564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,2,1,128,1,float16,fp8,0,0.06718400120735168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,2,1,128,1,float16,float16,0,0.45395731925964355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,2,2,128,1,float16,float16,0,0.33583998680114746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,2,2,128,1,float16,float16,0,0.07262399792671204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,2,2,128,1,float16,fp8,0,0.07257600128650665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,2,1,128,1,float16,fp8,0,0.4442826509475708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,2,2,128,1,float16,fp8,0,0.31060800949732464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,2,1,128,1,float16,float16,0,0.249834676583608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,2,1,128,1,float16,fp8,0,0.24562132358551025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,2,2,128,1,float16,fp8,0,0.3081706762313843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,2,2,128,1,float16,float16,0,0.20010133584340414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,2,2,128,1,float16,float16,0,0.34376001358032227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,2,2,128,1,float16,fp8,0,0.1785973310470581
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,2,1,128,1,float16,float16,0,0.14286933342615762
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,2,1,128,1,float16,fp8,0,0.1402293344338735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,2,2,128,1,float16,float16,0,0.20171733697255453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,2,2,128,1,float16,fp8,0,0.18007999658584595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,2,2,128,1,float16,float16,0,0.11375466982523601
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,2,2,128,1,float16,fp8,0,0.09759466846783955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,2,1,128,1,float16,float16,0,0.08269333342711131
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,2,1,128,1,float16,fp8,0,0.08683733145395915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,2,2,128,1,float16,float16,0,0.11197333534558614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,2,2,128,1,float16,fp8,0,0.09723732868830363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,2,2,128,1,float16,float16,0,0.06509333352247874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,2,2,128,1,float16,fp8,0,0.0666293352842331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,2,1,128,1,float16,float16,0,0.057248001297314964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,2,1,128,1,float16,fp8,0,0.05948266883691152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,2,2,128,1,float16,float16,0,0.06505066653092702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,2,2,128,1,float16,fp8,0,0.06694399813810985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,2,2,128,1,float16,float16,0,0.05761066575845083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,2,2,128,1,float16,fp8,0,0.05917333563168844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,2,1,128,1,float16,float16,0,0.05312533179918925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,2,1,128,1,float16,fp8,0,0.055120001236597695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,2,2,128,1,float16,float16,0,0.05787733197212219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,2,2,128,1,float16,fp8,0,0.05914666752020518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,2,1,128,1,float16,float16,0,0.47940266132354736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,2,2,128,1,float16,float16,0,0.37352001667022705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,2,1,128,1,float16,fp8,0,0.4834933280944824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,2,1,128,1,float16,float16,0,0.25805866718292236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,2,2,128,1,float16,fp8,0,0.34229334195454914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,2,1,128,1,float16,fp8,0,0.26080532868703205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,2,2,128,1,float16,float16,0,0.3846026659011841
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,2,2,128,1,float16,float16,0,0.22198933362960815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,2,2,128,1,float16,fp8,0,0.3412319819132487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,2,2,128,1,float16,fp8,0,0.19101333618164062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,2,1,128,1,float16,fp8,0,0.14809067050615946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,2,1,128,1,float16,float16,0,0.1458346645037333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,2,2,128,1,float16,fp8,0,0.19056532780329385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,2,2,128,1,float16,float16,0,0.22590400775273642
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,2,2,128,1,float16,fp8,0,0.10899200042088826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,2,2,128,1,float16,float16,0,0.1263146698474884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,2,1,128,1,float16,float16,0,0.08307733138402303
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,2,1,128,1,float16,fp8,0,0.08275199929873149
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,2,2,128,1,float16,float16,0,0.12527466813723245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,2,2,128,1,float16,fp8,0,0.10870400071144104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,2,2,128,1,float16,float16,0,0.05794133245944977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,2,2,128,1,float16,fp8,0,0.057520002126693726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,2,1,128,1,float16,float16,0,0.04700266818205515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,2,1,128,1,float16,fp8,0,0.04850666721661886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,2,2,128,1,float16,float16,0,0.05796800057093302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,2,2,128,1,float16,fp8,0,0.05734399954477946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,2,2,128,1,float16,float16,0,0.047744000951449074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,2,2,128,1,float16,fp8,0,0.04862933357556661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,2,1,128,1,float16,float16,0,0.042303999265034996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,2,1,128,1,float16,fp8,0,0.04398933549722036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,2,2,128,1,float16,float16,0,0.047237331668535866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,2,2,128,1,float16,fp8,0,0.04894933104515076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,2,2,128,1,float16,float16,0,0.04215999941031138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,2,2,128,1,float16,fp8,0,0.04413333535194397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,2,1,128,1,float16,float16,0,0.039317332208156586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,2,1,128,1,float16,fp8,0,0.04030933231115341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,2,2,128,1,float16,float16,0,0.0421013335386912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,2,2,128,1,float16,fp8,0,0.04398933549722036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,2,1,128,1,float16,float16,0,0.3295946717262268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,2,2,128,1,float16,float16,0,0.27811199426651
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,2,1,128,1,float16,fp8,0,0.33585600058237713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,2,1,128,1,float16,float16,0,0.18233599265416464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,2,2,128,1,float16,fp8,0,0.2487786610921224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,2,1,128,1,float16,fp8,0,0.18617600202560425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,2,2,128,1,float16,float16,0,0.2791733344395955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,2,2,128,1,float16,fp8,0,0.24732800324757895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,2,2,128,1,float16,float16,0,0.16048533717791238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,2,2,128,1,float16,fp8,0,0.14170666535695395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,2,1,128,1,float16,float16,0,0.10199466347694397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,2,2,128,1,float16,float16,0,0.16024000446001688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,2,1,128,1,float16,fp8,0,0.10574932893117268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,2,2,128,1,float16,fp8,0,0.14339733123779297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,2,2,128,1,float16,float16,0,0.08528000116348267
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,2,2,128,1,float16,fp8,0,0.07866666714350383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,2,2,128,1,float16,fp8,0,0.07586666444937389
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,2,1,128,1,float16,float16,0,0.056847999493281044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,2,1,128,1,float16,fp8,0,0.06273599962393443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,2,2,128,1,float16,float16,0,0.08377599716186523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,2,2,128,1,float16,float16,0,0.04621866842110952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,2,1,128,1,float16,float16,0,0.03812800099452337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,2,2,128,1,float16,fp8,0,0.05012266834576925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,2,1,128,1,float16,fp8,0,0.04215999941031138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,2,2,128,1,float16,float16,0,0.04628799855709076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,2,2,128,1,float16,fp8,0,0.04955733319123586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,2,2,128,1,float16,float16,0,0.03866666555404663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,2,2,128,1,float16,fp8,0,0.04197333256403605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,2,1,128,1,float16,float16,0,0.03477866699298223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,2,1,128,1,float16,fp8,0,0.037989333271980286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,2,2,128,1,float16,float16,0,0.03884266565243403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,2,2,128,1,float16,fp8,0,0.042352000872294106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,2,2,128,1,float16,float16,0,0.034815999368826546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,2,2,128,1,float16,fp8,0,0.038378665844599404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,2,1,128,1,float16,float16,0,0.03326933334271113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,2,1,128,1,float16,fp8,0,0.035973332822322845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,2,2,128,1,float16,float16,0,0.034671999514102936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,2,2,128,1,float16,fp8,0,0.038586666186650596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,2,1,128,1,float16,float16,0,0.376911997795105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,2,2,128,1,float16,float16,0,0.3118613362312317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,2,1,128,1,float16,fp8,0,0.3952266772588094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,2,1,128,1,float16,float16,0,0.20148799816767374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,2,2,128,1,float16,fp8,0,0.29517332712809247
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,2,1,128,1,float16,fp8,0,0.21119467417399088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,2,2,128,1,float16,float16,0,0.31883732477823895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,2,2,128,1,float16,fp8,0,0.29498666524887085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,2,2,128,1,float16,fp8,0,0.16491732994715372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,2,2,128,1,float16,float16,0,0.190010666847229
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,2,1,128,1,float16,fp8,0,0.12001066406567891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,2,2,128,1,float16,float16,0,0.18923733631769815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,2,2,128,1,float16,fp8,0,0.1644480029741923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,2,1,128,1,float16,float16,0,0.11179199814796448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,2,2,128,1,float16,float16,0,0.10416533549626668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,2,2,128,1,float16,fp8,0,0.09198400378227234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,2,1,128,1,float16,float16,0,0.059205333391825356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,2,2,128,1,float16,float16,0,0.10558399558067322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,2,1,128,1,float16,fp8,0,0.06378666559855144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,2,2,128,1,float16,fp8,0,0.0940106709798177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,2,2,128,1,float16,float16,0,0.045279999574025474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,2,2,128,1,float16,fp8,0,0.04814933240413666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,2,1,128,1,float16,float16,0,0.03425599883000056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,2,1,128,1,float16,fp8,0,0.037903999288876854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,2,2,128,1,float16,float16,0,0.04534400006135305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,2,2,128,1,float16,fp8,0,0.04807466765244802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,2,2,128,1,float16,float16,0,0.034714666505654655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,2,2,128,1,float16,fp8,0,0.03953066716591517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,2,1,128,1,float16,float16,0,0.029722665747006733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,2,1,128,1,float16,fp8,0,0.033258666594823204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,2,2,128,1,float16,float16,0,0.03461866577466329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,2,2,128,1,float16,fp8,0,0.038959999879201256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,2,2,128,1,float16,float16,0,0.029743999242782593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,2,2,128,1,float16,fp8,0,0.034143999218940735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,2,1,128,1,float16,float16,0,0.02720000098148982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,2,1,128,1,float16,fp8,0,0.030266667405764263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,2,2,128,1,float16,float16,0,0.029765332738558452
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,2,2,128,1,float16,fp8,0,0.03430933256944021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,2,2,128,1,float16,float16,0,0.026693334182103474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,2,2,128,1,float16,fp8,0,0.030368000268936157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,2,1,128,1,float16,float16,0,0.025285333395004272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,2,1,128,1,float16,fp8,0,0.028016000986099243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,2,2,128,1,float16,float16,0,0.026880001028378803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,2,2,128,1,float16,fp8,0,0.030058667063713074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,2,2,128,1,float16,float16,0,0.271397332350413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,2,1,128,1,float16,float16,0,0.28957333167394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,2,2,128,1,float16,fp8,0,0.26105066140492755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,2,1,128,1,float16,float16,0,0.15667200088500977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,2,1,128,1,float16,fp8,0,0.3255786697069804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,2,1,128,1,float16,fp8,0,0.17765865723292032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,2,2,128,1,float16,float16,0,0.2687893311182658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,2,2,128,1,float16,float16,0,0.16662399967511496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,2,2,128,1,float16,fp8,0,0.26211732625961304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,2,2,128,1,float16,fp8,0,0.14812800288200378
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,2,1,128,1,float16,float16,0,0.08919466535250346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,2,1,128,1,float16,fp8,0,0.10194133718808492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,2,2,128,1,float16,float16,0,0.1680319905281067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,2,2,128,1,float16,float16,0,0.08578667044639587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,2,2,128,1,float16,fp8,0,0.14752533038457236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,2,2,128,1,float16,float16,0,0.0909493366877238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,2,2,128,1,float16,fp8,0,0.08137600123882294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,2,1,128,1,float16,float16,0,0.045509333411852516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,2,1,128,1,float16,fp8,0,0.05426666637261709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,2,2,128,1,float16,fp8,0,0.08142933249473572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,2,1,128,1,float16,fp8,0,0.03416533271471659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,2,2,128,1,float16,float16,0,0.038773333032925926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,2,2,128,1,float16,fp8,0,0.04275733232498169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,2,1,128,1,float16,float16,0,0.027786667148272198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,2,2,128,1,float16,float16,0,0.03798400113979975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,2,2,128,1,float16,fp8,0,0.042837331692377724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,2,2,128,1,float16,float16,0,0.02842666705449422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,2,2,128,1,float16,fp8,0,0.03482666611671448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,2,1,128,1,float16,float16,0,0.023205332458019257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,2,1,128,1,float16,fp8,0,0.030181333422660828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,2,2,128,1,float16,float16,0,0.028624000648657482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,2,2,128,1,float16,fp8,0,0.034346667428811394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,2,2,128,1,float16,float16,0,0.02349333216746648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,2,2,128,1,float16,float16,0,0.020549333343903225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,2,2,128,1,float16,fp8,0,0.029653333127498627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,2,1,128,1,float16,float16,0,0.02063999945918719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,2,1,128,1,float16,float16,0,0.019786667078733444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,2,1,128,1,float16,fp8,0,0.026101333399613697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,2,2,128,1,float16,float16,0,0.023621333142121632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,2,2,128,1,float16,fp8,0,0.02518400053183238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,2,2,128,1,float16,fp8,0,0.030031998952229817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,2,2,128,1,float16,fp8,0,0.025936000049114227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,2,1,128,1,float16,fp8,0,0.02380266785621643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,2,2,128,1,float16,float16,0,0.02086399992307027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,2,2,128,1,float16,float16,0,0.01894933357834816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,2,2,128,1,float16,fp8,0,0.023599999646345775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,2,1,128,1,float16,float16,0,0.018976000448067982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,2,1,128,1,float16,fp8,0,0.02330133318901062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,2,2,128,1,float16,float16,0,0.018800000349680584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,2,2,128,1,float16,fp8,0,0.023472001155217487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,2,2,128,1,float16,float16,0,0.1595306694507599
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,2,2,128,1,float16,fp8,0,0.1364479959011078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,2,1,128,1,float16,float16,0,0.14523200194040933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,2,1,128,1,float16,float16,0,0.08183999856313069
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,2,1,128,1,float16,fp8,0,0.1639199952284495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,2,1,128,1,float16,fp8,0,0.09405333797136943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,2,2,128,1,float16,float16,0,0.15958933035532633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,2,2,128,1,float16,float16,0,0.08540266752243042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,2,2,128,1,float16,fp8,0,0.13581867019335428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,2,2,128,1,float16,fp8,0,0.07348266740640004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,2,1,128,1,float16,float16,0,0.04058133314053217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,2,1,128,1,float16,fp8,0,0.0499946673711141
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,2,2,128,1,float16,float16,0,0.08533333738644917
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,2,2,128,1,float16,fp8,0,0.07482133309046428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,2,2,128,1,float16,float16,0,0.035877334574858345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,2,2,128,1,float16,fp8,0,0.03976533313592275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,2,1,128,1,float16,float16,0,0.02454400062561035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,2,1,128,1,float16,fp8,0,0.030037333567937214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,2,2,128,1,float16,float16,0,0.035317334036032356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,2,2,128,1,float16,fp8,0,0.03967999915281931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,2,2,128,1,float16,float16,0,0.02587199956178665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,2,2,128,1,float16,fp8,0,0.03153600047032038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,2,1,128,1,float16,float16,0,0.02054399996995926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,2,1,128,1,float16,fp8,0,0.02587733417749405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,2,2,128,1,float16,float16,0,0.025466665625572205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,2,2,128,1,float16,fp8,0,0.03141333411137263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,2,2,128,1,float16,float16,0,0.020608000457286835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,2,2,128,1,float16,fp8,0,0.026752000053723652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,2,1,128,1,float16,float16,0,0.0182239996890227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,2,1,128,1,float16,fp8,0,0.02250133454799652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,2,2,128,1,float16,float16,0,0.02067199970285098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,2,2,128,1,float16,fp8,0,0.026421333352724712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,2,2,128,1,float16,float16,0,0.01756799966096878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,2,2,128,1,float16,fp8,0,0.02179733415444692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,2,1,128,1,float16,float16,0,0.01642666632930438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,2,1,128,1,float16,fp8,0,0.019808000574509304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,2,2,128,1,float16,float16,0,0.017610666652520496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,2,2,128,1,float16,fp8,0,0.021738665799299877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,2,2,128,1,float16,float16,0,0.01578666642308235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,2,2,128,1,float16,fp8,0,0.01958400011062622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,2,2,128,1,float16,fp8,0,0.019658666104078293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,2,1,128,1,float16,float16,0,0.016069332758585613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,2,1,128,1,float16,fp8,0,0.01941866676012675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,2,2,128,1,float16,float16,0,0.01609066625436147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,2,2,128,1,float16,float16,0,0.01534933348496755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,2,2,128,1,float16,fp8,0,0.018976000448067982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,2,2,128,1,float16,fp8,0,0.019391999890406925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,2,1,128,1,float16,float16,0,0.015583999454975128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,2,1,128,1,float16,fp8,0,0.018885333091020584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,2,2,128,1,float16,float16,0,0.015626666446526844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,2,2,128,1,float16,float16,0,0.08332799871762593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,2,1,128,1,float16,float16,0,0.07932800054550171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,2,1,128,1,float16,fp8,0,0.08455466230710347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,2,2,128,1,float16,fp8,0,0.06770666440327962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,2,1,128,1,float16,float16,0,0.03809066613515218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,2,1,128,1,float16,fp8,0,0.04368533194065094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,2,2,128,1,float16,float16,0,0.08390933275222778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,2,2,128,1,float16,float16,0,0.03414933383464813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,2,2,128,1,float16,fp8,0,0.06735466420650482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,2,2,128,1,float16,fp8,0,0.035946667194366455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,2,1,128,1,float16,float16,0,0.023365333676338196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,2,1,128,1,float16,fp8,0,0.027024000883102417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,2,2,128,1,float16,float16,0,0.034389334420363106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,2,2,128,1,float16,fp8,0,0.035631999373435974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,2,2,128,1,float16,float16,0,0.02420799930890401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,2,2,128,1,float16,float16,0,0.024143998821576435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,2,2,128,1,float16,fp8,0,0.0273333340883255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,2,1,128,1,float16,float16,0,0.019215999792019527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,2,1,128,1,float16,fp8,0,0.02250133454799652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,2,2,128,1,float16,fp8,0,0.02720000098148982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,2,2,128,1,float16,float16,0,0.018965333700180054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,2,2,128,1,float16,fp8,0,0.02233600119749705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,2,1,128,1,float16,float16,0,0.016714667280515034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,2,1,128,1,float16,fp8,0,0.0194560003777345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,2,2,128,1,float16,float16,0,0.019189332922299702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,2,2,128,1,float16,fp8,0,0.022448000808556873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,2,2,128,1,float16,float16,0,0.016143999993801117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,2,2,128,1,float16,fp8,0,0.018885333091020584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,2,1,128,1,float16,float16,0,0.015103999525308609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,2,1,128,1,float16,fp8,0,0.01782400036851565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,2,2,128,1,float16,float16,0,0.01591466615597407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,2,2,128,1,float16,fp8,0,0.018698666244745255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,2,2,128,1,float16,float16,0,0.014245333770910898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,2,2,128,1,float16,fp8,0,0.01632000009218852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,2,1,128,1,float16,float16,0,0.014453332871198654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,2,1,128,1,float16,fp8,0,0.016362667083740234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,2,2,128,1,float16,float16,0,0.01440000037352244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,2,2,128,1,float16,fp8,0,0.016704000532627106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,2,2,128,1,float16,float16,0,0.013973332941532135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,2,2,128,1,float16,fp8,0,0.01632533346613248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,2,1,128,1,float16,float16,0,0.013914667069911957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,2,1,128,1,float16,fp8,0,0.016058667252461117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,2,2,128,1,float16,fp8,0,0.01613866661985715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,2,2,128,1,float16,float16,0,0.01413333291808764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,2,2,128,1,float16,float16,0,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,2,2,128,1,float16,fp8,0,0.01590399940808614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,2,1,128,1,float16,float16,0,0.013914667069911957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,2,1,128,1,float16,fp8,0,0.016234666109085083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,2,2,128,1,float16,float16,0,0.013765333841244379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,2,2,128,1,float16,fp8,0,0.01594666639963786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,2,2,128,1,float16,float16,0,0.03548266738653183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,2,1,128,1,float16,float16,0,0.03931200007597605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,2,2,128,1,float16,fp8,0,0.03525333354870478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,2,1,128,1,float16,float16,0,0.02363733450571696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,2,1,128,1,float16,fp8,0,0.042303999265034996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,2,1,128,1,float16,fp8,0,0.026687999566396076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,2,2,128,1,float16,float16,0,0.022431999444961548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,2,2,128,1,float16,float16,0,0.035989334185918175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,2,2,128,1,float16,fp8,0,0.03514133393764496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,2,2,128,1,float16,fp8,0,0.02345066765944163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,2,1,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,2,1,128,1,float16,fp8,0,0.0225600004196167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,2,2,128,1,float16,float16,0,0.022218666970729828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,2,2,128,1,float16,fp8,0,0.02342933416366577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,2,2,128,1,float16,float16,0,0.01717866708834966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,2,2,128,1,float16,fp8,0,0.018661333868900936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,2,1,128,1,float16,float16,0,0.01651200031240781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,2,1,128,1,float16,fp8,0,0.019167999426523846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,2,2,128,1,float16,float16,0,0.01718933383623759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,2,2,128,1,float16,fp8,0,0.01860800012946129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,2,2,128,1,float16,float16,0,0.014639999717473984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,2,2,128,1,float16,fp8,0,0.01545599972208341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,2,1,128,1,float16,float16,0,0.01522133375207583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,2,1,128,1,float16,fp8,0,0.016864000509182613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,2,2,128,1,float16,float16,0,0.014645333091417948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,2,2,128,1,float16,fp8,0,0.015413332730531693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,2,2,128,1,float16,float16,0,0.012805332740147909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,2,2,128,1,float16,fp8,0,0.013573333621025085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,2,1,128,1,float16,float16,0,0.014309333016475042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,2,1,128,1,float16,float16,0,0.01368533323208491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,2,1,128,1,float16,fp8,0,0.01611199975013733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,2,2,128,1,float16,float16,0,0.013274667163689932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,2,2,128,1,float16,fp8,0,0.013157332936922709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,2,2,128,1,float16,float16,0,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,2,2,128,1,float16,fp8,0,0.012784000486135483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,2,1,128,1,float16,fp8,0,0.016122666498025257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,2,2,128,1,float16,float16,0,0.012293333808581034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,2,2,128,1,float16,fp8,0,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,2,2,128,1,float16,float16,0,0.011952000359694162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,2,2,128,1,float16,fp8,0,0.012527999778588613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,2,1,128,1,float16,float16,0,0.013781332721312841
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,2,1,128,1,float16,fp8,0,0.01589866727590561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,2,2,128,1,float16,float16,0,0.011855999628702799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,2,2,128,1,float16,fp8,0,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,2,2,128,1,float16,float16,0,0.011898666620254517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,2,2,128,1,float16,fp8,0,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,2,1,128,1,float16,float16,0,0.014080000420411428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,2,1,128,1,float16,fp8,0,0.01571200042963028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,2,2,128,1,float16,float16,0,0.012053333222866058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,2,2,128,1,float16,fp8,0,0.012282667060693106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,2,2,128,1,float16,float16,0,0.025706666211287182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,2,1,128,1,float16,float16,0,0.02514133354028066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,2,2,128,1,float16,fp8,0,0.026965332527955372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,2,1,128,1,float16,fp8,0,0.026575999955336254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,2,1,128,1,float16,float16,0,0.017477333545684814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,2,1,128,1,float16,fp8,0,0.018826667219400406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,2,2,128,1,float16,float16,0,0.0259253333012263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,2,2,128,1,float16,fp8,0,0.026778665681680042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,2,2,128,1,float16,float16,0,0.017429333180189133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,2,2,128,1,float16,fp8,0,0.01883200059334437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,2,1,128,1,float16,float16,0,0.014853333433469137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,2,1,128,1,float16,fp8,0,0.01579733317097028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,2,2,128,1,float16,float16,0,0.01739199956258138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,2,1,128,1,float16,float16,0,0.013754667093356451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,2,2,128,1,float16,fp8,0,0.01859733338157336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,2,2,128,1,float16,float16,0,0.014549333602190018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,2,2,128,1,float16,fp8,0,0.015615999698638916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,2,1,128,1,float16,fp8,0,0.013983999689420065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,2,2,128,1,float16,float16,0,0.014554666976133982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,2,2,128,1,float16,fp8,0,0.015669333438078564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,2,2,128,1,float16,float16,0,0.013359999905029932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,2,2,128,1,float16,fp8,0,0.01351999988158544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,2,1,128,1,float16,float16,0,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,2,1,128,1,float16,fp8,0,0.013269333789745966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,2,2,128,1,float16,float16,0,0.013151999562978745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,2,2,128,1,float16,fp8,0,0.01360000049074491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,2,2,128,1,float16,float16,0,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,2,2,128,1,float16,fp8,0,0.013295999417702356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,2,1,128,1,float16,float16,0,0.012186666329701742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,2,1,128,1,float16,fp8,0,0.01293333371480306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,2,2,128,1,float16,float16,0,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,2,2,128,1,float16,fp8,0,0.013050666699806849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,2,2,128,1,float16,float16,0,0.012080000092585882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,2,2,128,1,float16,fp8,0,0.012645332763592402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,2,2,128,1,float16,float16,0,0.011792000383138657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,2,1,128,1,float16,float16,0,0.012223999947309494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,2,1,128,1,float16,fp8,0,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,2,2,128,1,float16,float16,0,0.012223999947309494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,2,2,128,1,float16,fp8,0,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,2,2,128,1,float16,fp8,0,0.012335999558369318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,2,1,128,1,float16,float16,0,0.011514666179815928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,2,1,128,1,float16,fp8,0,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,2,2,128,1,float16,float16,0,0.011701333026091257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,2,2,128,1,float16,fp8,0,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,2,2,128,1,float16,float16,0,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,2,2,128,1,float16,fp8,0,0.012319999436537424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,2,1,128,1,float16,float16,0,0.011887999872366587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,2,1,128,1,float16,fp8,0,0.012389333297808966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,2,2,128,1,float16,float16,0,0.011509332805871964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,2,2,128,1,float16,fp8,0,0.012367999802033106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,2,2,128,1,float16,float16,0,0.021007999777793884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,2,2,128,1,float16,fp8,0,0.02233600119749705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,2,1,128,1,float16,float16,0,0.021066665649414062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,2,1,128,1,float16,float16,0,0.015295999745527903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,2,1,128,1,float16,fp8,0,0.022250667214393616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,2,1,128,1,float16,fp8,0,0.015872000406185787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,2,2,128,1,float16,fp8,0,0.02232533444960912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,2,2,128,1,float16,fp8,0,0.01544533297419548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,2,1,128,1,float16,float16,0,0.013744000345468521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,2,2,128,1,float16,float16,0,0.021082667013009388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,2,2,128,1,float16,float16,0,0.014896000425020853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,2,1,128,1,float16,fp8,0,0.01431999976436297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,2,2,128,1,float16,float16,0,0.014975999792416891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,2,2,128,1,float16,float16,0,0.013301332791646322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,2,2,128,1,float16,fp8,0,0.015439999600251516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,2,2,128,1,float16,fp8,0,0.013834666460752487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,2,1,128,1,float16,float16,0,0.01321600005030632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,2,1,128,1,float16,fp8,0,0.013317332913478216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,2,2,128,1,float16,float16,0,0.013210666676362356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,2,2,128,1,float16,fp8,0,0.013797332843144735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,2,2,128,1,float16,float16,0,0.013327999661366144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,2,2,128,1,float16,fp8,0,0.013514666507641474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,2,1,128,1,float16,float16,0,0.012538666526476542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,2,1,128,1,float16,fp8,0,0.01292266696691513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,2,2,128,1,float16,float16,0,0.013258667041858038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,2,2,128,1,float16,fp8,0,0.013434667140245438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,2,2,128,1,float16,float16,0,0.012335999558369318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,2,2,128,1,float16,fp8,0,0.012757333616415659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,2,1,128,1,float16,float16,0,0.012138667205969492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,2,1,128,1,float16,fp8,0,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,2,2,128,1,float16,float16,0,0.012650666137536367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,2,2,128,1,float16,fp8,0,0.012938667088747025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,2,2,128,1,float16,float16,0,0.012186666329701742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,2,2,128,1,float16,fp8,0,0.012629333883523941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,2,1,128,1,float16,float16,0,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,2,1,128,1,float16,fp8,0,0.012351999680201212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,2,2,128,1,float16,float16,0,0.011887999872366587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,2,2,128,1,float16,fp8,0,0.012549333274364471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,2,2,128,1,float16,float16,0,0.011658667276302973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,2,2,128,1,float16,fp8,0,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,2,1,128,1,float16,float16,0,0.011727999895811081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,2,1,128,1,float16,fp8,0,0.012373333175977072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,2,2,128,1,float16,float16,0,0.011519999553759893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,2,2,128,1,float16,fp8,0,0.01231466606259346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,2,2,128,1,float16,float16,0,0.011792000383138657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,2,2,128,1,float16,fp8,0,0.012213333199421564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,2,1,128,1,float16,float16,0,0.011525332927703857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,2,1,128,1,float16,fp8,0,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,2,2,128,1,float16,float16,0,0.011770666887362799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,2,2,128,1,float16,fp8,0,0.012410666793584824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,1,1,128,1,float16,float16,0,0.6298506657282511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,1,1,128,1,float16,fp8,0,0.6485866705576578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,1,1,128,1,float16,float16,0,0.6198879877726237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,1,1,128,1,float16,fp8,0,0.6503359874089559
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,1,1,128,1,float16,float16,0,0.379749337832133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,1,1,128,1,float16,fp8,0,0.4002506732940674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,1,1,128,1,float16,float16,0,0.3823253313700358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,1,1,128,1,float16,fp8,0,0.4014666477839152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,1,1,128,1,float16,float16,0,0.2269973357518514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,1,1,128,1,float16,fp8,0,0.2421706716219584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,1,1,128,1,float16,float16,0,0.2294666568438212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,1,1,128,1,float16,fp8,0,0.24130666255950928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,1,1,128,1,float16,float16,0,0.41174399852752686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,1,1,128,1,float16,fp8,0,0.3840479850769043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,1,1,128,1,float16,float16,0,0.2524213393529256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,1,1,128,1,float16,float16,0,0.4186026652654012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,1,1,128,1,float16,fp8,0,0.3832266728083293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,1,1,128,1,float16,float16,0,0.253930668036143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,1,1,128,1,float16,fp8,0,0.23033066590627035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,1,1,128,1,float16,fp8,0,0.228767991065979
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,1,1,128,1,float16,float16,0,0.17683732509613037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,1,1,128,1,float16,fp8,0,0.16501333316167197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,1,1,128,1,float16,float16,0,0.17529600858688354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,1,1,128,1,float16,fp8,0,0.1649493376413981
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,1,1,128,1,float16,float16,0,0.31597334146499634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,1,1,128,1,float16,fp8,0,0.2950826684633891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,1,1,128,1,float16,float16,0,0.3164373238881429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,1,1,128,1,float16,fp8,0,0.29702399174372357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,1,1,128,1,float16,fp8,0,0.1779413421948751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,1,1,128,1,float16,float16,0,0.18850666284561157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,1,1,128,1,float16,fp8,0,0.1763733426729838
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,1,1,128,1,float16,float16,0,0.1900213360786438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,1,1,128,1,float16,float16,0,0.1479520003000895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,1,1,128,1,float16,float16,0,0.38186665376027423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,1,1,128,1,float16,fp8,0,0.14154666662216187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,1,1,128,1,float16,float16,0,0.14904533823331198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,1,1,128,1,float16,fp8,0,0.14190399646759033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,1,1,128,1,float16,fp8,0,0.36138665676116943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,1,1,128,1,float16,float16,0,0.3848373492558797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,1,1,128,1,float16,fp8,0,0.3630719979604085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,1,1,128,1,float16,float16,0,0.23577600717544556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,1,1,128,1,float16,fp8,0,0.2194933295249939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,1,1,128,1,float16,float16,0,0.23484800259272257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,1,1,128,1,float16,float16,0,0.12070932984352112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,1,1,128,1,float16,fp8,0,0.22123199701309204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,1,1,128,1,float16,float16,0,0.13075199723243713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,1,1,128,1,float16,fp8,0,0.12601600090662637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,1,1,128,1,float16,float16,0,0.1309599975744883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,1,1,128,1,float16,fp8,0,0.12627733747164407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,1,1,128,1,float16,fp8,0,0.1169653336207072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,1,1,128,1,float16,float16,0,0.12095466256141663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,1,1,128,1,float16,fp8,0,0.24462932348251343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,1,1,128,1,float16,fp8,0,0.11726400256156921
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,1,1,128,1,float16,float16,0,0.25573867559432983
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,1,1,128,1,float16,fp8,0,0.2463573416074117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,1,1,128,1,float16,float16,0,0.25648534297943115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,1,1,128,1,float16,float16,0,0.15391467014948526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,1,1,128,1,float16,fp8,0,0.14426666498184204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,1,1,128,1,float16,float16,0,0.15311466654141745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,1,1,128,1,float16,fp8,0,0.1437493364016215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,1,1,128,1,float16,float16,0,0.10082133611043294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,1,1,128,1,float16,fp8,0,0.09878399968147278
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,1,1,128,1,float16,float16,0,0.10051199793815613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,1,1,128,1,float16,fp8,0,0.10028266906738281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,1,1,128,1,float16,float16,0,0.09355733791987102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,1,1,128,1,float16,fp8,0,0.09165866176287334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,1,1,128,1,float16,fp8,0,0.24804800748825073
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,1,1,128,1,float16,float16,0,0.09384533762931824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,1,1,128,1,float16,fp8,0,0.09156800309816997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,1,1,128,1,float16,float16,0,0.252618670463562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,1,1,128,1,float16,float16,0,0.2521653374036153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,1,1,128,1,float16,float16,0,0.1511306663354238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,1,1,128,1,float16,fp8,0,0.2465653419494629
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,1,1,128,1,float16,fp8,0,0.1437279979387919
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,1,1,128,1,float16,float16,0,0.15128533045450845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,1,1,128,1,float16,fp8,0,0.14450666308403015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,1,1,128,1,float16,float16,0,0.08037333190441132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,1,1,128,1,float16,fp8,0,0.08082666496435802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,1,1,128,1,float16,float16,0,0.07905599971612294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,1,1,128,1,float16,fp8,0,0.08071466783682506
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,1,1,128,1,float16,float16,0,0.070933332045873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,1,1,128,1,float16,fp8,0,0.07190399865309398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,1,1,128,1,float16,float16,0,0.07141333321730296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,1,1,128,1,float16,fp8,0,0.07277866701285045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,1,1,128,1,float16,float16,0,0.06642666459083557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,1,1,128,1,float16,fp8,0,0.06660266717274983
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,1,1,128,1,float16,fp8,0,0.1762453317642212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,1,1,128,1,float16,float16,0,0.06669333577156067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,1,1,128,1,float16,fp8,0,0.06713599960009257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,1,1,128,1,float16,float16,0,0.10100266337394714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,1,1,128,1,float16,float16,0,0.1754186749458313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,1,1,128,1,float16,float16,0,0.1743839979171753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,1,1,128,1,float16,fp8,0,0.17543466885884604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,1,1,128,1,float16,fp8,0,0.10083199540774028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,1,1,128,1,float16,float16,0,0.0988106628259023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,1,1,128,1,float16,fp8,0,0.09860266248385112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,1,1,128,1,float16,float16,0,0.06310933331648509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,1,1,128,1,float16,fp8,0,0.06568000217278798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,1,1,128,1,float16,float16,0,0.06260266900062561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,1,1,128,1,float16,fp8,0,0.06560533245404561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,1,1,128,1,float16,float16,0,0.056746666630109154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,1,1,128,1,float16,fp8,0,0.05898133416970571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,1,1,128,1,float16,float16,0,0.05677866439024607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,1,1,128,1,float16,fp8,0,0.05894400179386139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,1,1,128,1,float16,float16,0,0.05351466437180837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,1,1,128,1,float16,fp8,0,0.05481066803137461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,1,1,128,1,float16,float16,0,0.05386666456858317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,1,1,128,1,float16,fp8,0,0.0553706685702006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,1,1,128,1,float16,float16,0,0.18467199802398682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,1,1,128,1,float16,fp8,0,0.18756266434987387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,1,1,128,1,float16,float16,0,0.10671466588973999
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,1,1,128,1,float16,fp8,0,0.1881386637687683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,1,1,128,1,float16,float16,0,0.18406933546066284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,1,1,128,1,float16,fp8,0,0.10613866647084554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,1,1,128,1,float16,float16,0,0.10775466759999593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,1,1,128,1,float16,fp8,0,0.1048906644185384
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,1,1,128,1,float16,float16,0,0.05379733443260193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,1,1,128,1,float16,fp8,0,0.05625600119431814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,1,1,128,1,float16,float16,0,0.0537120004494985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,1,1,128,1,float16,fp8,0,0.056517332792282104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,1,1,128,1,float16,float16,0,0.04626133541266123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,1,1,128,1,float16,fp8,0,0.04821333289146423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,1,1,128,1,float16,float16,0,0.04598933458328247
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,1,1,128,1,float16,fp8,0,0.04799466828505198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,1,1,128,1,float16,float16,0,0.041738669077555336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,1,1,128,1,float16,fp8,0,0.04398933549722036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,1,1,128,1,float16,float16,0,0.038912000755469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,1,1,128,1,float16,float16,0,0.042170668641726174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,1,1,128,1,float16,fp8,0,0.04363200068473816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,1,1,128,1,float16,float16,0,0.03945599993069967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,1,1,128,1,float16,fp8,0,0.03965333352486292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,1,1,128,1,float16,fp8,0,0.03994666785001755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,1,1,128,1,float16,float16,0,0.132042666276296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,1,1,128,1,float16,fp8,0,0.13844799995422363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,1,1,128,1,float16,fp8,0,0.1395146648089091
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,1,1,128,1,float16,float16,0,0.13217066725095114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,1,1,128,1,float16,float16,0,0.07592533528804779
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,1,1,128,1,float16,fp8,0,0.07588799794514973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,1,1,128,1,float16,float16,0,0.07627200086911519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,1,1,128,1,float16,fp8,0,0.07458133498827617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,1,1,128,1,float16,float16,0,0.04373333354791006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,1,1,128,1,float16,fp8,0,0.04915200173854828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,1,1,128,1,float16,float16,0,0.04419733087221781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,1,1,128,1,float16,fp8,0,0.04905599852403005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,1,1,128,1,float16,float16,0,0.03794133414824804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,1,1,128,1,float16,fp8,0,0.04172799984614054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,1,1,128,1,float16,float16,0,0.038032000263532005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,1,1,128,1,float16,fp8,0,0.04191466669241587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,1,1,128,1,float16,float16,0,0.03444266567627589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,1,1,128,1,float16,fp8,0,0.0384853333234787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,1,1,128,1,float16,float16,0,0.03451200077931086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,1,1,128,1,float16,fp8,0,0.03851733356714249
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,1,1,128,1,float16,float16,0,0.03229333211978277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,1,1,128,1,float16,fp8,0,0.035045333206653595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,1,1,128,1,float16,float16,0,0.0324799989660581
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,1,1,128,1,float16,fp8,0,0.03505599995454153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,1,1,128,1,float16,float16,0,0.15025066335995993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,1,1,128,1,float16,fp8,0,0.16165332992871603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,1,1,128,1,float16,float16,0,0.14949333667755127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,1,1,128,1,float16,float16,0,0.08691199620564778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,1,1,128,1,float16,fp8,0,0.16244799892107645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,1,1,128,1,float16,fp8,0,0.09062400460243225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,1,1,128,1,float16,float16,0,0.0845973292986552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,1,1,128,1,float16,fp8,0,0.0897759993871053
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,1,1,128,1,float16,float16,0,0.04160533348719279
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,1,1,128,1,float16,fp8,0,0.047354668378829956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,1,1,128,1,float16,float16,0,0.041237334410349526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,1,1,128,1,float16,fp8,0,0.04685866832733154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,1,1,128,1,float16,float16,0,0.03389866650104523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,1,1,128,1,float16,fp8,0,0.03878399978081385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,1,1,128,1,float16,float16,0,0.03350933392842611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,1,1,128,1,float16,fp8,0,0.03846399982770284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,1,1,128,1,float16,float16,0,0.029552000264326733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,1,1,128,1,float16,fp8,0,0.033999999364217125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,1,1,128,1,float16,float16,0,0.029301332930723827
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,1,1,128,1,float16,fp8,0,0.03410666684309641
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,1,1,128,1,float16,float16,0,0.02700799951950709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,1,1,128,1,float16,fp8,0,0.030250666042168934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,1,1,128,1,float16,float16,0,0.02644266684850057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,1,1,128,1,float16,fp8,0,0.030106666187445324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,1,1,128,1,float16,float16,0,0.02548266698916753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,1,1,128,1,float16,fp8,0,0.029391999046007793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,1,1,128,1,float16,float16,0,0.02518933266401291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,1,1,128,1,float16,fp8,0,0.028533334533373516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,1,1,128,1,float16,float16,0,0.1255573332309723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,1,1,128,1,float16,float16,0,0.07170133292675018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,1,1,128,1,float16,fp8,0,0.14230400323867798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,1,1,128,1,float16,float16,0,0.12633066376050314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,1,1,128,1,float16,fp8,0,0.1437440017859141
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,1,1,128,1,float16,fp8,0,0.0783679982026418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,1,1,128,1,float16,float16,0,0.07296533385912578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,1,1,128,1,float16,fp8,0,0.07784000039100647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,1,1,128,1,float16,float16,0,0.034373333056767784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,1,1,128,1,float16,fp8,0,0.04160533348719279
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,1,1,128,1,float16,float16,0,0.03395200024048487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,1,1,128,1,float16,fp8,0,0.04148799926042557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,1,1,128,1,float16,float16,0,0.027242665489514668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,1,1,128,1,float16,fp8,0,0.033759998778502144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,1,1,128,1,float16,float16,0,0.027274665733178455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,1,1,128,1,float16,fp8,0,0.03355200091997782
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,1,1,128,1,float16,float16,0,0.023599999646345775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,1,1,128,1,float16,fp8,0,0.02940266579389572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,1,1,128,1,float16,float16,0,0.02306666721900304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,1,1,128,1,float16,fp8,0,0.029919999341169994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,1,1,128,1,float16,float16,0,0.020224000016848247
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,1,1,128,1,float16,fp8,0,0.02550933261712392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,1,1,128,1,float16,float16,0,0.0201706662774086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,1,1,128,1,float16,fp8,0,0.025386666258176167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,1,1,128,1,float16,float16,0,0.019088000059127808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,1,1,128,1,float16,fp8,0,0.02390933285156886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,1,1,128,1,float16,float16,0,0.019226666539907455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,1,1,128,1,float16,fp8,0,0.02366400013367335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,1,1,128,1,float16,float16,0,0.018485333770513535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,1,1,128,1,float16,fp8,0,0.02319466571013133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,1,1,128,1,float16,float16,0,0.01855466639002164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,1,1,128,1,float16,fp8,0,0.023200000325838726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,1,1,128,1,float16,float16,0,0.0676746666431427
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,1,1,128,1,float16,fp8,0,0.06962133447329204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,1,1,128,1,float16,float16,0,0.06644266843795776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,1,1,128,1,float16,float16,0,0.031717332700888314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,1,1,128,1,float16,fp8,0,0.06968533496061961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,1,1,128,1,float16,fp8,0,0.03845866769552231
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,1,1,128,1,float16,float16,0,0.03148799886306127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,1,1,128,1,float16,fp8,0,0.03846399982770284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,1,1,128,1,float16,float16,0,0.024341332415739696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,1,1,128,1,float16,fp8,0,0.03035199890534083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,1,1,128,1,float16,float16,0,0.024373332659403484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,1,1,128,1,float16,fp8,0,0.03052799900372823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,1,1,128,1,float16,float16,0,0.020549333343903225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,1,1,128,1,float16,fp8,0,0.026416001220544178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,1,1,128,1,float16,float16,0,0.0205226664741834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,1,1,128,1,float16,fp8,0,0.026538667579491932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,1,1,128,1,float16,float16,0,0.01748266691962878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,1,1,128,1,float16,fp8,0,0.021946666141351063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,1,1,128,1,float16,fp8,0,0.02102400114138921
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,1,1,128,1,float16,float16,0,0.01800000046690305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,1,1,128,1,float16,fp8,0,0.022015998760859173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,1,1,128,1,float16,float16,0,0.016186666985352833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,1,1,128,1,float16,float16,0,0.016058667252461117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,1,1,128,1,float16,fp8,0,0.019808000574509304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,1,1,128,1,float16,float16,0,0.015429332852363586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,1,1,128,1,float16,fp8,0,0.019285333653291065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,1,1,128,1,float16,float16,0,0.015637333194414776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,1,1,128,1,float16,fp8,0,0.019519999623298645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,1,1,128,1,float16,float16,0,0.015301333119471868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,1,1,128,1,float16,fp8,0,0.019061333189407986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,1,1,128,1,float16,float16,0,0.015599999576807022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,1,1,128,1,float16,fp8,0,0.018992000569899876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,1,1,128,1,float16,float16,0,0.030282666285832722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,1,1,128,1,float16,fp8,0,0.03430933256944021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,1,1,128,1,float16,float16,0,0.029738667110602062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,1,1,128,1,float16,float16,0,0.022874665757020313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,1,1,128,1,float16,fp8,0,0.03442133218050003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,1,1,128,1,float16,fp8,0,0.026693334182103474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,1,1,128,1,float16,float16,0,0.023007998863856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,1,1,128,1,float16,fp8,0,0.026330667237440746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,1,1,128,1,float16,float16,0,0.01899733394384384
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,1,1,128,1,float16,fp8,0,0.021957332889238994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,1,1,128,1,float16,float16,0,0.018906666586796444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,1,1,128,1,float16,fp8,0,0.021957332889238994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,1,1,128,1,float16,float16,0,0.016330666840076447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,1,1,128,1,float16,fp8,0,0.018794666975736618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,1,1,128,1,float16,float16,0,0.015957333147525787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,1,1,128,1,float16,fp8,0,0.01836799954374631
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,1,1,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,1,1,128,1,float16,fp8,0,0.016682667036851246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,1,1,128,1,float16,float16,0,0.014501333236694336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,1,1,128,1,float16,fp8,0,0.016538667182127636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,1,1,128,1,float16,float16,0,0.014245333770910898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,1,1,128,1,float16,fp8,0,0.016255999604860943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,1,1,128,1,float16,float16,0,0.013989333063364029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,1,1,128,1,float16,fp8,0,0.016000000139077503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,1,1,128,1,float16,float16,0,0.013376000026861826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,1,1,128,1,float16,fp8,0,0.01594666639963786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,1,1,128,1,float16,float16,0,0.013893333574136099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,1,1,128,1,float16,fp8,0,0.01594666639963786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,1,1,128,1,float16,float16,0,0.013744000345468521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,1,1,128,1,float16,fp8,0,0.015925332903862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,1,1,128,1,float16,float16,0,0.013967999567588171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,1,1,128,1,float16,fp8,0,0.01586666703224182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,1,1,128,1,float16,float16,0,0.02146133283774058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,1,1,128,1,float16,fp8,0,0.022976001103719074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,1,1,128,1,float16,fp8,0,0.022511998812357586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,1,1,128,1,float16,float16,0,0.021402666966120403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,1,1,128,1,float16,float16,0,0.01729600007335345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,1,1,128,1,float16,fp8,0,0.018645333747069042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,1,1,128,1,float16,float16,0,0.01711999997496605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,1,1,128,1,float16,fp8,0,0.018618666877349217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,1,1,128,1,float16,float16,0,0.014661333213249842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,1,1,128,1,float16,fp8,0,0.015173333386580149
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,1,1,128,1,float16,float16,0,0.014549333602190018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,1,1,128,1,float16,fp8,0,0.015605332950750986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,1,1,128,1,float16,float16,0,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,1,1,128,1,float16,fp8,0,0.013306666165590286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,1,1,128,1,float16,float16,0,0.013077333569526672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,1,1,128,1,float16,fp8,0,0.013429333766301474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,1,1,128,1,float16,float16,0,0.012213333199421564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,1,1,128,1,float16,fp8,0,0.012815999488035837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,1,1,128,1,float16,float16,0,0.012362666428089142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,1,1,128,1,float16,fp8,0,0.01309866706530253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,1,1,128,1,float16,float16,0,0.011722666521867117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,1,1,128,1,float16,fp8,0,0.012186666329701742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,1,1,128,1,float16,float16,0,0.012074666718641916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,1,1,128,1,float16,fp8,0,0.012432000289360682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,1,1,128,1,float16,float16,0,0.011898666620254517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,1,1,128,1,float16,fp8,0,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,1,1,128,1,float16,float16,0,0.011882666498422623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,1,1,128,1,float16,fp8,0,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,1,1,128,1,float16,float16,0,0.011685332904259363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,1,1,128,1,float16,fp8,0,0.012608000387748083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,1,1,128,1,float16,float16,0,0.011829332758982977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,1,1,128,1,float16,fp8,0,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,1,1,128,1,float16,float16,0,0.017370666066805523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,1,1,128,1,float16,fp8,0,0.018895999838908512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,1,1,128,1,float16,float16,0,0.01728533332546552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,1,1,128,1,float16,float16,0,0.014773332824309668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,1,1,128,1,float16,fp8,0,0.018672000616788864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,1,1,128,1,float16,fp8,0,0.015482666591803232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,1,1,128,1,float16,float16,0,0.01444799949725469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,1,1,128,1,float16,fp8,0,0.015365333606799444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,1,1,128,1,float16,float16,0,0.01360000049074491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,1,1,128,1,float16,fp8,0,0.013967999567588171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,1,1,128,1,float16,fp8,0,0.012954667210578918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,1,1,128,1,float16,float16,0,0.013343999783198038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,1,1,128,1,float16,fp8,0,0.014261333892742792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,1,1,128,1,float16,float16,0,0.012442667037248611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,1,1,128,1,float16,float16,0,0.012997332960367203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,1,1,128,1,float16,float16,0,0.012437333663304647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,1,1,128,1,float16,fp8,0,0.013066666821638743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,1,1,128,1,float16,float16,0,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,1,1,128,1,float16,fp8,0,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,1,1,128,1,float16,fp8,0,0.014815999815861383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,1,1,128,1,float16,float16,0,0.01163200040658315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,1,1,128,1,float16,fp8,0,0.012661332885424295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,1,1,128,1,float16,float16,0,0.011610666910807291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,1,1,128,1,float16,fp8,0,0.012304000556468964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,1,1,128,1,float16,float16,0,0.011786667009194693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,1,1,128,1,float16,fp8,0,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,1,1,128,1,float16,float16,0,0.011727999895811081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,1,1,128,1,float16,fp8,0,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,1,1,128,1,float16,float16,0,0.011584000041087469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,1,1,128,1,float16,fp8,0,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,1,1,128,1,float16,float16,0,0.011418666690587997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,1,1,128,1,float16,fp8,0,0.012272000312805176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,1,1,128,1,float16,float16,0,0.015263999501864115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,1,1,128,1,float16,fp8,0,0.016021333634853363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,1,1,128,1,float16,float16,0,0.013594667116800943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,1,1,128,1,float16,float16,0,0.015008000036080679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,1,1,128,1,float16,fp8,0,0.015941333025693893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,1,1,128,1,float16,fp8,0,0.014064000298579534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,1,1,128,1,float16,float16,0,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,1,1,128,1,float16,fp8,0,0.01379199946920077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,1,1,128,1,float16,float16,0,0.012960000584522883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,1,1,128,1,float16,fp8,0,0.013178666432698568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,1,1,128,1,float16,fp8,0,0.013327999661366144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,1,1,128,1,float16,float16,0,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,1,1,128,1,float16,fp8,0,0.013839999834696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,1,1,128,1,float16,float16,0,0.012533333152532578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,1,1,128,1,float16,float16,0,0.012096000214417776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,1,1,128,1,float16,fp8,0,0.01268799975514412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,1,1,128,1,float16,float16,0,0.012149333953857422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,1,1,128,1,float16,fp8,0,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,1,1,128,1,float16,float16,0,0.012026666353146235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,1,1,128,1,float16,fp8,0,0.012549333274364471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,1,1,128,1,float16,float16,0,0.011717333147923151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,1,1,128,1,float16,fp8,0,0.012629333883523941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,1,1,128,1,float16,float16,0,0.011477333803971609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,1,1,128,1,float16,fp8,0,0.012213333199421564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,1,1,128,1,float16,float16,0,0.011701333026091257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,1,1,128,1,float16,fp8,0,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,1,1,128,1,float16,float16,0,0.01184533288081487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,1,1,128,1,float16,fp8,0,0.012357333054145178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,1,1,128,1,float16,float16,0,0.011610666910807291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,1,1,128,1,float16,fp8,0,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,1,1,128,1,float16,float16,0,0.011663999408483505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,1,1,128,1,float16,fp8,0,0.012144000579913458
